{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tensorflow.keras.layers import *\n",
    "import tensorflow.keras.backend as K\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = \"0\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 加载数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('criteo_sampled_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>I1</th>\n",
       "      <th>I2</th>\n",
       "      <th>I3</th>\n",
       "      <th>I4</th>\n",
       "      <th>I5</th>\n",
       "      <th>I6</th>\n",
       "      <th>I7</th>\n",
       "      <th>I8</th>\n",
       "      <th>I9</th>\n",
       "      <th>...</th>\n",
       "      <th>C17</th>\n",
       "      <th>C18</th>\n",
       "      <th>C19</th>\n",
       "      <th>C20</th>\n",
       "      <th>C21</th>\n",
       "      <th>C22</th>\n",
       "      <th>C23</th>\n",
       "      <th>C24</th>\n",
       "      <th>C25</th>\n",
       "      <th>C26</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1382.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>181.0</td>\n",
       "      <td>...</td>\n",
       "      <td>e5ba7672</td>\n",
       "      <td>f54016b9</td>\n",
       "      <td>21ddcdc9</td>\n",
       "      <td>b1252a9d</td>\n",
       "      <td>07b5194c</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3a171ecb</td>\n",
       "      <td>c5c50484</td>\n",
       "      <td>e8b83407</td>\n",
       "      <td>9727dd16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>07c540c4</td>\n",
       "      <td>b04e4670</td>\n",
       "      <td>21ddcdc9</td>\n",
       "      <td>5840adea</td>\n",
       "      <td>60f6221e</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3a171ecb</td>\n",
       "      <td>43f13e8b</td>\n",
       "      <td>e8b83407</td>\n",
       "      <td>731c3655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>767.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>245.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8efede7f</td>\n",
       "      <td>3412118d</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>e587c466</td>\n",
       "      <td>ad3062eb</td>\n",
       "      <td>3a171ecb</td>\n",
       "      <td>3b183c5c</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>893</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4392.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1e88c74f</td>\n",
       "      <td>74ef3502</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6b3a5ca6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3a171ecb</td>\n",
       "      <td>9117a34a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1e88c74f</td>\n",
       "      <td>26b3c7a7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21c9516a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32c7478e</td>\n",
       "      <td>b34f3128</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   label   I1   I2    I3    I4      I5    I6    I7   I8     I9  ...       C17  \\\n",
       "0      0  1.0    1   5.0   0.0  1382.0   4.0  15.0  2.0  181.0  ...  e5ba7672   \n",
       "1      0  2.0    0  44.0   1.0   102.0   8.0   2.0  2.0    4.0  ...  07c540c4   \n",
       "2      0  2.0    0   1.0  14.0   767.0  89.0   4.0  2.0  245.0  ...  8efede7f   \n",
       "3      0  NaN  893   NaN   NaN  4392.0   NaN   0.0  0.0    0.0  ...  1e88c74f   \n",
       "4      0  3.0   -1   NaN   0.0     2.0   0.0   3.0  0.0    0.0  ...  1e88c74f   \n",
       "\n",
       "        C18       C19       C20       C21       C22       C23       C24  \\\n",
       "0  f54016b9  21ddcdc9  b1252a9d  07b5194c       NaN  3a171ecb  c5c50484   \n",
       "1  b04e4670  21ddcdc9  5840adea  60f6221e       NaN  3a171ecb  43f13e8b   \n",
       "2  3412118d       NaN       NaN  e587c466  ad3062eb  3a171ecb  3b183c5c   \n",
       "3  74ef3502       NaN       NaN  6b3a5ca6       NaN  3a171ecb  9117a34a   \n",
       "4  26b3c7a7       NaN       NaN  21c9516a       NaN  32c7478e  b34f3128   \n",
       "\n",
       "        C25       C26  \n",
       "0  e8b83407  9727dd16  \n",
       "1  e8b83407  731c3655  \n",
       "2       NaN       NaN  \n",
       "3       NaN       NaN  \n",
       "4       NaN       NaN  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = data.columns.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据预处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义特征组"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dense_feats = [f for f in cols if f[0] == \"I\"]\n",
    "sparse_feats = [f for f in cols if f[0] == \"C\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 处理dense特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_dense_feats(data, feats):\n",
    "    d = data.copy()\n",
    "    d = d[feats].fillna(0.0)\n",
    "    for f in feats:\n",
    "        d[f] = d[f].apply(lambda x: np.log(x+1) if x > -1 else -1)\n",
    "    \n",
    "    return d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dense = process_dense_feats(data, dense_feats)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 处理sparse特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_sparse_feats(data, feats):\n",
    "    d = data.copy()\n",
    "    d = d[feats].fillna(\"-1\")\n",
    "    for f in feats:\n",
    "        label_encoder = LabelEncoder()\n",
    "        d[f] = label_encoder.fit_transform(d[f])\n",
    "        \n",
    "    return d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_sparse = process_sparse_feats(data, sparse_feats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_data = pd.concat([data_dense, data_sparse], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_data['label'] = data['label']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AutoInt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## embedding层"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### dense特征"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在AutoInt中，dense特征也一样做了embedding，并用原始的dense值对embedding进行了scale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "k=8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "dense_inputs = []\n",
    "for f in dense_feats:\n",
    "    _input = Input([1], name=f)\n",
    "    dense_inputs.append(_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'I1:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I2:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I3:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I4:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I5:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I6:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I7:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I8:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I9:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I10:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I11:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I12:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I13:0' shape=(None, 1) dtype=float32>]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dense_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "dense_kd_embed = []\n",
    "for i, _input in enumerate(dense_inputs):\n",
    "    f = dense_feats[i]\n",
    "    embed = tf.Variable(tf.random.truncated_normal(shape=(1, k), stddev=0.01), name=f)\n",
    "    \n",
    "    scaled_embed = tf.expand_dims(_input * embed, axis=1)\n",
    "\n",
    "    dense_kd_embed.append(scaled_embed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tf.expand_dims([[1,2,3,4,5,6,7,8]],axis=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'ExpandDims:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'ExpandDims_1:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'ExpandDims_2:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'ExpandDims_3:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'ExpandDims_4:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'ExpandDims_5:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'ExpandDims_6:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'ExpandDims_7:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'ExpandDims_8:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'ExpandDims_9:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'ExpandDims_10:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'ExpandDims_11:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'ExpandDims_12:0' shape=(None, 1, 8) dtype=float32>]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dense_kd_embed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### sparse特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "sparse_inputs = []\n",
    "for f in sparse_feats:\n",
    "    _input = Input([1], name=f)\n",
    "    sparse_inputs.append(_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "sparse_kd_embed = []\n",
    "for i, _input in enumerate(sparse_inputs):\n",
    "    f = sparse_feats[i]\n",
    "    voc_size = data[f].nunique()\n",
    "    _embed = Embedding(voc_size+1, k, embeddings_regularizer=tf.keras.regularizers.l2(0.5))(_input)\n",
    "    sparse_kd_embed.append(_embed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Embedding(voc_size+1,k,embeddings_regularizer=tf.keras.regularizers.l2(0.5))(_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'embedding/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_1/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_2/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_3/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_4/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_5/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_6/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_7/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_8/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_9/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_10/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_11/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_12/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_13/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_14/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_15/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_16/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_17/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_18/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_19/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_20/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_21/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_22/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_23/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_24/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_25/Identity:0' shape=(None, 1, 8) dtype=float32>]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparse_kd_embed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 合并embedding层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_embeds = dense_kd_embed + sparse_kd_embed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'ExpandDims:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'ExpandDims_1:0' shape=(None, 1, 8) dtype=float32>]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_embeds[:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Interacting Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建feature map\n",
    "embed_map = Concatenate(axis=1)(input_embeds)  # ?, 39, 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'concatenate/Identity:0' shape=(None, 39, 8) dtype=float32>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embed_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def auto_interacting(embed_map, d=6, n_attention_head=2):\n",
    "    \"\"\"\n",
    "    实现单层 AutoInt Interacting Layer\n",
    "    @param embed_map: 输入的embedding feature map, (?, n_feats, n_dim)\n",
    "    @param d: Q,K,V映射后的维度\n",
    "    @param n_attention_head: multi-head attention的个数\n",
    "    \"\"\"\n",
    "    assert len(embed_map.shape) == 3, \"Input embedding feature map must be 3-D tensor.\"\n",
    "    \n",
    "    k = embed_map.shape[-1]\n",
    "    \n",
    "    # 存储多个self-attention的结果\n",
    "    attention_heads = []\n",
    "    W_Q = []\n",
    "    W_K = []\n",
    "    W_V = []\n",
    "    \n",
    "    # 1.构建多个attention\n",
    "    for i in range(n_attention_head):\n",
    "        # 初始化W_Q, W_K, W_V\n",
    "        W_Q.append(tf.Variable(tf.random.truncated_normal(shape=(k, d)), name=\"query_\"+str(i)))  # k, d 8，6\n",
    "        W_K.append(tf.Variable(tf.random.truncated_normal(shape=(k, d)), name=\"key_\"+str(i)))  # k, d\n",
    "        W_V.append(tf.Variable(tf.random.truncated_normal(shape=(k, d)), name=\"value_\"+str(i)))  # k, d\n",
    "     \n",
    "    for i in range(n_attention_head):\n",
    "        # 映射到d维空间\n",
    "        embed_q = tf.matmul(embed_map, W_Q[i])  # ?, 39, d\n",
    "        embed_k = tf.matmul(embed_map, W_K[i])  # ?, 39, d\n",
    "        embed_v = tf.matmul(embed_map, W_V[i])  # ?, 39, d\n",
    "    \n",
    "        # 计算attention\n",
    "        energy = tf.matmul(embed_q, tf.transpose(embed_k, [0, 2, 1]))  # ?, 39, 39\n",
    "        attention = tf.nn.softmax(energy)  # ?, 39, 39\n",
    "    \n",
    "        attention_output = tf.matmul(attention, embed_v)  # ?, 39, d\n",
    "        attention_heads.append(attention_output)\n",
    "    \n",
    "    # 2.concat multi head\n",
    "    multi_attention_output = Concatenate(axis=-1)(attention_heads)  # ?, 39, n_attention_head*d\n",
    "    \n",
    "    # 3.ResNet\n",
    "    w_res = tf.Variable(tf.random.truncated_normal(shape=(k, d*n_attention_head)), name=\"w_res_\"+str(i))  # k, d*n_attention_head\n",
    "    output = Activation(\"relu\")(multi_attention_output + tf.matmul(embed_map, w_res))  # ?, 39, d*n_attention_head)\n",
    "    \n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_autoint(x0, n_layers):\n",
    "    xl = x0\n",
    "    for i in range(n_layers):\n",
    "        xl = auto_interacting(xl, d=6, n_attention_head=2)\n",
    "    \n",
    "    return xl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建3层interacting layer\n",
    "autoint_layer = build_autoint(embed_map, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "autoint_layer = Flatten()(autoint_layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'flatten/Identity:0' shape=(None, 468) dtype=float32>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "autoint_layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 输出层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_layer = Dense(1, activation=\"sigmoid\")(autoint_layer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 编译模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.utils import plot_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Model(dense_inputs+sparse_inputs, output_layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed to import pydot. You must install pydot and graphviz for `pydotprint` to work.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "\n",
    "plot_model(model, \"autoint.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer=\"adam\", \n",
    "              loss=\"binary_crossentropy\", \n",
    "              metrics=[\"binary_crossentropy\", tf.keras.metrics.AUC(name='auc')])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.callbacks import TensorBoard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tbCallBack = TensorBoard(log_dir='./logs',  # log 目录\n",
    "#                  histogram_freq=0, \n",
    "#                  write_graph=True,  \n",
    "#                  write_grads=True,\n",
    "#                  write_images=True,\n",
    "#                  embeddings_freq=0, \n",
    "#                  embeddings_layer_names=None, \n",
    "#                  embeddings_metadata=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = total_data.loc[:500000-1]\n",
    "valid_data = total_data.loc[500000:]\n",
    "\n",
    "train_dense_x = [train_data[f].values for f in dense_feats]\n",
    "train_sparse_x = [train_data[f].values for f in sparse_feats]\n",
    "\n",
    "train_label = [train_data['label'].values]\n",
    "\n",
    "val_dense_x = [valid_data[f].values for f in dense_feats]\n",
    "val_sparse_x = [valid_data[f].values for f in sparse_feats]\n",
    "val_label = [valid_data['label'].values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([0, 0, 0, ..., 0, 0, 1], dtype=int64)]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_dense_x+train_sparse_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 500000 samples, validate on 100000 samples\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n",
      "D:\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500000/500000 [==============================] - 115s 230us/sample - loss: 21.6660 - binary_crossentropy: 0.5077 - auc: 0.7223 - val_loss: 0.6212 - val_binary_crossentropy: 0.5102 - val_auc: 0.7236\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1efcb1cdc08>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(train_dense_x+train_sparse_x, \n",
    "          train_label, epochs=1, batch_size=256,\n",
    "          validation_data=(val_dense_x+val_sparse_x, val_label), \n",
    "         )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data[train_data.label==1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Attention可视化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "feats = dense_feats + sparse_feats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_data = list(train_data.loc[7][feats].values.reshape((-1,1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Softmax:0' shape=(None, 39, 39) dtype=float32>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers[100].get_output_at(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "attention_layer = Model(inputs=model.input,outputs=model.layers[102].get_output_at(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "ename": "FailedPreconditionError",
     "evalue": " Error while reading resource variable _AnonymousVar42 from Container: localhost. This could mean that the variable was uninitialized. Not found: Resource localhost/_AnonymousVar42/class tensorflow::Var does not exist.\n\t [[node MatMul_5/ReadVariableOp (defined at D:\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\ops.py:1751) ]] [Op:__inference_keras_scratch_graph_18848]\n\nFunction call stack:\nkeras_scratch_graph\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFailedPreconditionError\u001b[0m                   Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-49-2885a3b87e1f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mattention\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mattention_layer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msample_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mD:\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, x, batch_size, verbose, steps, callbacks, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m    907\u001b[0m         \u001b[0mmax_queue_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmax_queue_size\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    908\u001b[0m         \u001b[0mworkers\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mworkers\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 909\u001b[1;33m         use_multiprocessing=use_multiprocessing)\n\u001b[0m\u001b[0;32m    910\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    911\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0mreset_metrics\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training_arrays.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, model, x, batch_size, verbose, steps, callbacks, **kwargs)\u001b[0m\n\u001b[0;32m    720\u001b[0m         \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    721\u001b[0m         \u001b[0msteps\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msteps\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 722\u001b[1;33m         callbacks=callbacks)\n\u001b[0m",
      "\u001b[1;32mD:\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training_arrays.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[1;34m(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, initial_epoch, steps_per_epoch, validation_steps, validation_freq, mode, validation_in_fit, prepared_feed_values_from_dataset, steps_name, **kwargs)\u001b[0m\n\u001b[0;32m    391\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    392\u001b[0m         \u001b[1;31m# Get outputs.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 393\u001b[1;33m         \u001b[0mbatch_outs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    394\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    395\u001b[0m           \u001b[0mbatch_outs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\keras\\backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   3738\u001b[0m         \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtensor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3739\u001b[0m       \u001b[0mconverted_inputs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3740\u001b[1;33m     \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_graph_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mconverted_inputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3741\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3742\u001b[0m     \u001b[1;31m# EagerTensor.numpy() will often make a copy to ensure memory safety.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1079\u001b[0m       \u001b[0mTypeError\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mFor\u001b[0m \u001b[0minvalid\u001b[0m \u001b[0mpositional\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0mkeyword\u001b[0m \u001b[0margument\u001b[0m \u001b[0mcombinations\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1080\u001b[0m     \"\"\"\n\u001b[1;32m-> 1081\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1082\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1083\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, args, kwargs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1119\u001b[0m       raise TypeError(\"Keyword arguments {} unknown. Expected {}.\".format(\n\u001b[0;32m   1120\u001b[0m           list(kwargs.keys()), list(self._arg_keywords)))\n\u001b[1;32m-> 1121\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_flat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcaptured_inputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1122\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1123\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_filtered_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1222\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mexecuting_eagerly\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1223\u001b[0m       flat_outputs = forward_function.call(\n\u001b[1;32m-> 1224\u001b[1;33m           ctx, args, cancellation_manager=cancellation_manager)\n\u001b[0m\u001b[0;32m   1225\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1226\u001b[0m       \u001b[0mgradient_name\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_delayed_rewrite_functions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mregister\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    509\u001b[0m               \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    510\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"executor_type\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexecutor_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"config_proto\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 511\u001b[1;33m               ctx=ctx)\n\u001b[0m\u001b[0;32m    512\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    513\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[1;32mD:\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     65\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     66\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 67\u001b[1;33m     \u001b[0msix\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mraise_from\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_status_to_exception\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcode\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     68\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     69\u001b[0m     keras_symbolic_tensors = [\n",
      "\u001b[1;32mD:\\anaconda3\\lib\\site-packages\\six.py\u001b[0m in \u001b[0;36mraise_from\u001b[1;34m(value, from_value)\u001b[0m\n",
      "\u001b[1;31mFailedPreconditionError\u001b[0m:  Error while reading resource variable _AnonymousVar42 from Container: localhost. This could mean that the variable was uninitialized. Not found: Resource localhost/_AnonymousVar42/class tensorflow::Var does not exist.\n\t [[node MatMul_5/ReadVariableOp (defined at D:\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\ops.py:1751) ]] [Op:__inference_keras_scratch_graph_18848]\n\nFunction call stack:\nkeras_scratch_graph\n"
     ]
    }
   ],
   "source": [
    "attention = attention_layer.predict(sample_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [],
   "source": [
    "attention = attention.reshape((39, 39))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_attention(data):\n",
    "    fig = plt.figure(figsize=(15, 15))\n",
    "    ax = fig.add_subplot(111)\n",
    "    ax.set_yticks(range(len(feats)))\n",
    "    ax.set_yticklabels(feats)\n",
    "    ax.set_xticks(range(len(feats)))\n",
    "    ax.set_xticklabels(feats)\n",
    "    \n",
    "    im = ax.imshow(data)\n",
    "    plt.colorbar(im)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0kAAAM9CAYAAABJyjiXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzde5RdZ3nn+e9Ppbvki7CJryIQghto8CggExJijGFoLjNp4tU0mEkbSNMIkrAmmU7SzaR7rWGGZk3CfZik8YiGAQITMyQ4OCAwDDbEdDBBBl+QHYLtGJDlGAwYWxfrUvXMH3UqbMplWdK7j85W6ftZ66w69e59nv2cffY5p5563/3uVBWSJEmSpFlLJp2AJEmSJA2JRZIkSZIkdVgkSZIkSVKHRZIkSZIkdVgkSZIkSVLH0kknIEmSJOknPe/CNfX9H0xPOo2Duu7GvVdW1fMnncc4WCRJkiRJA/P9H0zzN1c+atJpHNTUGd88ddI5jIvD7SRJkiSpwyJJkiRJkjocbidJkiQNTAEzzEw6jeOWPUmSJEmS1GGRJEmSJEkdDreTJEmSBqeYLofbTYo9SZIkSZLUYZEkSZIkSR0Ot5MkSZIGZnZ2u5p0Gscte5IkSZIkqcMiSZIkSZI6LJIkSZIkqcNzkiRJkqQBmsEpwCfFniRJkiRJ6rBIkiRJkqQOh9tJkiRJA1MU0+UU4JNiT5IkSZIkdVgkSZIkSVKHw+0kSZKkAZrB4XaTYk+SJEmSJHVYJEmSJElSh8PtJEmSpIEpYNrhdhNjT5IkSZIkdVgkSZIkSVKHw+0kSZKkAXJ2u8mxJ0mSJEmSOiySJEmSJKnDIkmSJEmSOjwnSZIkSRqYAqbLc5ImxZ4kSZIkSeqwSJIkSZKkDofbSZIkSQM0M+kEjmP2JEmSJElSh0WSJEmSJHU43E6SJEkamKKYxtntJsWeJEmSJEnqsEiSJEmSpA6H20mSJElDUzDtaLuJsSdJkiRJkjoskiRJkiSpw+F2kiRJ0sAUXkx2kuxJkiRJkqQOiyRJkiRJ6nC4nSRJkjQ4YZpMOonjlj1JkiRJktRhkSRJkiRJHRZJkiRJktThOUmSJEnSwBQwU5PO4vhlT5IkSZIkdVgkSZIkSVKHw+0kSZKkAXIK8MmxJ0mSJEmSOiySJEmSJI1Fkucn+UaSW5O8foHlSfKu0fIbkzxl1L4+ydVJbkmyLclvdR7zhiR3Jrl+dHvhqP1XO23XJ5lJsiHJ6iSfTPK3o1h/8HB5O9xOkiRJGpji2B9ul2QK+GPgucB24CtJrqiqmzurvQB43Oj288C7Rz8PAL9TVV9NcgJwXZLPdh77jqp6a3d7VfVh4MOjbT8Z+HhVXZ9kNfDWqro6yXLgc0leUFWfeqjc7UmSJEmSNA5PA26tqturah9wGfCieeu8CPhgzboWODnJGVV1V1V9FaCq7gduAc46jG2/DPjT0eN3V9XVo/v7gK8CZx/swRZJkiRJksbhLOA7nd+38+BC52HXSfJo4OeAL3eaXzcanve+JOsW2PZLGRVJ82KdDPwy8LmDJW6RJEmSJA3QTGXQN+DUJFs7t03znsJC4wXnXyL3oOskWQv8OfDbVXXfqPndwGOBDcBdwNt+ImDy88Duqvr6vPalzBZO76qq2w+27z0nSZIkSdKRuKeqNh5k+XZgfef3s4Edh7pOkmXMFkgfrqqPza1QVXfP3U/yHuAT82JezAK9SMBm4JtV9c6D5AzYkyRJkiRpPL4CPC7JY0YTJlwMXDFvnSuAl49muXs68KOquitJgPcCt1TV27sPSHJG59eLgK93li0B/iWz5z91H/OfgJOA3z6UxO1JkiRJkgZmMcxuV1UHkrwOuBKYAt5XVduSvHa0/FJgC/BC4FZgN/Bro4c/A7gEuCnJ9aO236+qLcCbk2xgdjfdAbyms9lnAtu7w+mSnA38B+Bvga/O1l/8UVX9l4fKPVXzhwVKkiRJmqQnnru8PvSJ0yedxkE99ae/c93DDLc7ZjncTpIkSZI6LJIkSZIkqcNzkiRJkqSBKcK0/RkT456XJEmSpA6LJEmSJEnqcLidJEmSNEAzdWxPAX4ssydJkiRJkjoskiRJkiSpw+F2kiRJ0sAUMI3D7SbFniRJkiRJ6hhMT9LUmjW17ORHNMWoPp7NVPUQBJbsWTyV/0wP+3XJgfYYfcl0e4yZZT3kMdMeow99nBNaU+0xoJ/jpHr4108fx0gfr28fzwX6OV77+Gfmkr3tMfo41tLPx3wvhnK89vH6Duq7oo/P1z6+xvv4fO0jjx6OsyX722P0pfVY23/vD5jetWvx/KGmsRhMkbTs5Efw06/5t00x9p7awzfFif18Cqy6ZWUvcYZg76nt3zYr7hlOp+WKH7b/hbTn9PbP1j7+YOxDH/9c2HdSPxXfiu+3HycHVre/vsvva399l+5uDsGB1e0xAPac3v76zCxr368n3N5e4exf0xxiMP+gAJhZ0R5j6c4e8uihkN63rv0YWfH9fv5uneqjIO/ha2u6h9d3ZnkPeaxsf23W3NnPa9PHft3zyLbn853//I72JI6KMN3Xf8t02NzzkiRJktRhkSRJkiRJHYMZbidJkiRpVgEz9mdMjHtekiRJkjp6LZKS7Ozc/3SSe5N8os9tSJIkSdI4jbMn6S3AJWOML0mSJEm9G9s5SVX1uSTPGld8SZIkaTGb7uWCXToSEz0nKcmmJFuTbJ3etWuSqUiSJEkSMOEiqao2V9XGqto4taaHKwRKkiRJUiOnAJckSZIGpipMlxNRT4p7XpIkSZI6xtaTlOQa4PHA2iTbgVdV1ZXj2p4kSZIk9aHXIqmq1nbun99nbEmSJOl4MuPsdhPjcDtJkiRJ6rBIkiRJkqSOwcxut/y+Gc6+ek9TjNP+979vzuNDj/58cwyAc2/4jV7iDMHMqfuaY0ztWNlDJv04/ZofNMf4zgtPaY6RmeYQvVj+vWqOsfcRPSQCTLUfaux5wt7mGCddtaI5xrr3f6k5xvde+wvNMQB2PuFAc4xf+Ce3Ncf41hf/SXOMe85tH3qy4oftMQ709JH2qPO/3Rzjno+sb46x/4T2fbKk/a3HkvZDFYBl9/fwubaufZ/08Zm26rvtz2Xd/7C9OcaBN5/WHAPggUe0/+lZz/5R0+Ozaro5h6OhgGn7MybGPS9JkiRJHRZJkiRJktQxmOF2kiRJkuZ4MdlJcs9LkiRJUodFkiRJkiR19DrcLsnOqlqbZAPwbuBEYBp4U1V9pM9tSZIkSYtVATP2Z0zMuM5J2g28vKq+meRM4LokV1bVvWPaniRJkiT1YixFUlX9Xef+jiTfBR4JWCRJkiRJGrSxz26X5GnAcuBBVyJMsgnYBLBixUnjTkWSJEmSHtZYi6QkZwB/AryiqmbmL6+qzcBmgBNPOKv9ktKSJEnSIjFdmXQKx62xnQ2W5ETgk8B/rKprx7UdSZIkSerTWIqkJMuBy4EPVtVHx7ENSZIkSRqHcQ23ewnwTOCUJK8ctb2yqq4f0/YkSZKkRaMI004BPjG9FklVtXb080PAh/qMLUmSJElHg+WpJEmSJHWMfQpwSZIkSYdvpuzPmJTBFEl7T17C3//yiqYYv7Tmu815/Mzlr2mOAbBmqpcwg7D2xpXNMWpA+2P7805pjrHzsQeaY6y9fRhvv11ntk8vWqfs7SETqLvaj7WV32iPsfu05hDs/I+/2Bxj2f3teQBkZ/sbcOtfPb45xurT2o+1pbuaQ/TyebR0T3sMgB9+cH1zjP2n9vAe7mGfLPun97XncW0/10ycWdYeY9mu9iuT7D+h/bXZ+4j2GHde3X6crXhscwgAUu37dcn/t64th/sH9EeJBsvyVJIkSZI6hvGvbEmSJEn/qMDZ7SbIPS9JkiRJHRZJkiRJktThcDtJkiRpYIowXe0Td+jI9NqTlGTn6OdPJ7kuyfVJtiV5bZ/bkSRJkqRxGVdP0l3AL1bV3iRrga8nuaKqdoxpe5IkSZLUi7EUSVW1r/PrCjz3SZIkSdIxYmznJCVZD3wS+Fng9xbqRUqyCdgEMLWu7cJgkiRJ0mIyYz/DxIxtz1fVd6rqXGaLpFckedA17Ktqc1VtrKqNU2vWjCsVSZIkSTpkYy9PRz1I24Dzx70tSZIkSWo1luF2Sc4Gvl9Ve5KsA54BvH0c25IkSZIWmyqYLofbTcq4zkl6AvC2JAUEeGtV3TSmbUmSJElSb3otkqpq7ejnZ4Fz+4wtSZIkSUfD2Ga3kyRJknSkwgyZdBLHLQc6SpIkSVKHRZIkSZIkdQxmuF2mYfm9bTXbRz76rOY8VjZHmDW9vKdA6l8Pr82q7e1vnaEcI5lpj7Hy7/p55wxln8wsm3QGs/af0E+cld+d6idQo30nTTqDWdN97I6ejtWdqxfRUJqv9vAC97Rfp09dRPt1IPY+oq9Ik39tZobxkfiwCme3myT3vCRJkiR1WCRJkiRJUsdghttJkiRJ+rFp+zMmxj0vSZIkSR29FklJds77/cQkdyb5oz63I0mSJEnjMu7hdm8EvjDmbUiSJEmLShFmavKzAR6vxjbcLslTgdOAz4xrG5IkSZLUt7EUSUmWAG8Dfu9h1tuUZGuSrdO7d40jFUmSJEk6LOPqSfoNYEtVfedgK1XV5qraWFUbp1avGVMqkiRJknToxnVO0i8A5yf5DWAtsDzJzqp6/Zi2J0mSJC0qTgE+OWMpkqrqV+fuJ3klsNECSZIkSdKxwPJUkiRJkjp67UmqqrULtL0feH+f25EkSZIWswJmyv6MSXHPS5IkSVKHRZIkSZIkdYxrdjtJkiRJRyxMk0kncdyyJ0mSJEmSOiySJEmSJKnD4XaSJEnSwDi73WS55yVJkiSpwyJJkiRJkjp6HW6XZOfcBWWTTAM3jRZ9u6r+eZ/bkiRJkhYzZ7ebnHGek7SnqjaMMb4kSZIk9c7hdpIkSZLUMc4iaWWSrUmuTfIrY9yOJEmSJPVmnMPtHlVVO5L8DHBVkpuq6rbuCkk2AZsAlp60boypSJIkSceOqjgF+ASNbc9X1Y7Rz9uBzwM/t8A6m6tqY1VtnFq9ZlypSJIkSdIhG0uRlGRdkhWj+6cCzwBuHse2JEmSJKlP4xpu9wTg/0oyw2wh9gdVZZEkSZIkHaJph9tNTK9F0tw1kqrqr4En9xlbkiRJko4Gy1NJkiRJ6hjn7HaSJEmSjkABM2TSaRy37EmSJEmSpA6LJEmSJEljkeT5Sb6R5NYkr19geZK8a7T8xiRPGbWvT3J1kluSbEvyW53HvCHJnUmuH91eOGr/1U7b9UlmkmwYLXtqkptG23lXkoN20zncTpIkSRqcHPOz2yWZAv4YeC6wHfhKkivmzXr9AuBxo9vPA+8e/TwA/E5VfTXJCcB1ST7beew7quqt3e1V1YeBD4+2/WTg41V1/Wjxu4FNwLXAFuD5wKceKvdje89LkiRJGqqnAbdW1e1VtQ+4DHjRvHVeBHywZl0LnJzkjKq6q6q+ClBV9wO3AGcdxrZfBvwpQJIzgBOr6ktVVcAHgV852IMtkiRJkiQdiVOTbO3cNs1bfhbwnc7v23lwofOw6yR5NPBzwJc7za8bDc97X5J1C+T2UkZF0ije9ofJ4yc43E6SJEkamAJmavCz291TVRsPsnyhJ1CHs06StcCfA79dVfeNmt8NvHG03huBtwH/uvOYnwd2V9XXDyOPn2BPkiRJkqRx2A6s7/x+NrDjUNdJsozZAunDVfWxuRWq6u6qmq6qGeA9zA7r67qYH/cizW3j7IfJ4yf0WiQl2dm5/6gknxnNSHHzqJtMkiRJ0vHhK8DjkjwmyXJmi5cr5q1zBfDy0Sx3Twd+VFV3jWafey9wS1W9vfuA0TlGcy4Cvt5ZtgT4l8ye/wRAVd0F3J/k6aO4Lwc+frDExznc7oPAm6rqs6NuspkxbkuSJEnSgFTVgSSvA64EpoD3VdW2JK8dLb+U2ZnmXgjcCuwGfm308GcAlwA3JZmboe73q2oL8ObR1N4F3AG8prPZZwLbq+r2een8OvB+YBWzs9o95Mx2MKYiKckTgaVV9VmAqtr5MA+RJEmS1DG9CM6MGRU1W+a1Xdq5X8BvLvC4L7LwuURU1SUH2d7ngacv0L4VeNKh5j2uPX8OcG+SjyX5WpK3jOZJ/wlJNs3NhjG9e9eYUpEkSZKkQzeuImkpcD7wu8B5wM8Ar5y/UlVtrqqNVbVxavWaMaUiSZIkSYduXOckbQe+NjcWMMlfMNvt9d4xbU+SJElaNIocC1OAL1rj6kn6CrAuySNHvz8buHlM25IkSZKk3oylSKqqaWaH2n0uyU3MnnT1nnFsS5IkSZL61Otwu6pa27n/WeDcPuNLkiRJx4uZRTC73bHKPS9JkiRJHRZJkiRJktQxrtntjky1PXz/CY0BgEdsaw4BwP2PWjyzkaz6Xvt+3fPI4eyPJQfaY+x67P7mGKu+taw9kR4cWNP++u4/ebqHTGDVne0fSVN72/M40MMVCR44rf1AW/Ptfj6iD6xqj7Hqu+0x9j6iPUZ6eP9WD/8eTPvbBoA1d7YH2tnH981Me4hVT7+nOcaeL5/angiQPj6SetitfUxMNr2q/RhZfVd7InvXNYcA+nn/rf6Htn2ypP0r/Kiogmlnt5sYe5IkSZIkqcMiSZIkSZI6hjXcTpIkSRKAF5OdIHuSJEmSJKnDIkmSJEmSOnodbpdkZ1WtTXIh8I7OoscDF1fVX/S5PUmSJGkxKsJMH9MB6oiM5Zykqroa2ACQ5BHArcBnxrEtSZIkSerT0ShPXwx8qqp2H4VtSZIkSVKTo1EkXQz86UILkmxKsjXJ1uldu45CKpIkSZJ0cGOdAjzJGcCTgSsXWl5Vm4HNACvPXN/T9cslSZKkY980TgE+KePuSXoJcHlV7R/zdiRJkiSpF+Mukl7GQwy1kyRJkqQhGttwuySPBtYDXxjXNiRJkqTFqICZcrjdpPRaJFXV2s79O4Cz+owvSZIkSePmFaokSZIkqWOss9tJkiRJOhJhpuzPmJRhFUmNwy6X7Wwft3n/TzeHWHT2/NTiGg87s6w9xqpv9xBkILt16e72RJbu7umjpId9Mr2yhzSm22Os2tG+T2Z62q1LephfdO+69hj0cKGHmmqP0Ye+ThPYub6HQH1cQKOHNPZ8+dT2ID0ZynHSh6kH2l+cXt6/PclMe4zWv0v6+DtAi5/lqSRJkiR1DKsnSZIkSRIAM0MZdnIcsidJkiRJkjoskiRJkiSpw+F2kiRJ0sBUwbQXk52YXnuSkuzs3H9zkm1JbknyriS+ypIkSZIGbyzD7ZL8IvAM4FzgScB5wAXj2JYkSZIk9Wlcw+0KWAksZ/bqC8uAu8e0LUmSJEnqzViKpKr6UpKrgbuYLZL+qKpumb9ekk3AJoClJw3oSmeSJEnShM2Uc6xNyriG2/0s8ATgbOAs4NlJnjl/varaXFUbq2rj1Oo140hFkiRJkg7LuMrTi4Brq2pnVe0EPgU8fUzbkiRJkqTejKtI+jZwQZKlSZYxO2nDg4bbSZIkSXqwIszUsG+L2biKpD8DbgNuAm4AbqiqvxzTtiRJkiSpN71O3FBVa0c/p4HX9BlbkiRJko6GcU0BLkmSJKnBDIt7SNuQOa+gJEmSJHVYJEmSJElSh8PtJEmSpIEpWPQzyA2ZPUmSJEmS1GGRJEmSJEkdDreTJEmSBmim7M+YFPe8JEmSJHX0WiQl2dm5/4dJvj66vbTP7UiSJEnSuIxluF2S/w54CrABWAF8Icmnquq+cWxPkiRJWlQqzm43QeMabvdE4AtVdaCqdgE3AM8f07YkSZIkqTfjKpJuAF6QZHWSU4ELgfXzV0qyKcnWJFund+8aUyqSJEmSdOjGMtyuqj6T5Dzgr4HvAV8CDiyw3mZgM8DKM9fXOHKRJEmSpMMxtinAq+pNwJsAkvw/wDfHtS1JkiRpMSlgBs9JmpSxDLdLMpXklNH9c4Fzgc+MY1uSJEmS1Kdx9SQtA65JAnAf8K+q6kHD7SRJkiRpaHotkqpq7ejnA8zOcCdJkiTpCDgF+OSMa3Y7SZIkSTomWSRJkiRJUsfYZreTJEmSdGQKh9tN0rCKpMYrJS19oD2F+5+wrz0IsOqO5b3EGYI9Z7XPubHqzuEcalN722Os+GH7Zb12nz6MD74l+9tjLOvpWtB7T26PcWBN+2uz/Eftr83qf2jPY+f6fo6R6mHMwN5TpptjrL5zqjnG9MrmEKT9qfSyTwEeOL3983X1d9o/X6v9pWHdN9p37A/P6SERIDPtMXp5jft4C/dwFcm9p7bvkFX/0M9Bnx6ez66z2461Gs6fJBowh9tJkiRJUoe1tCRJkjRADrebHHuSJEmSJKnDIkmSJEmSOg67SEqys3P/00nuTfKJees8JsmXk3wzyUeSLJ5ZDCRJkqQxK8JMDfu2mLX2JL0FuGSB9j8E3lFVjwN+CLyqcTuSJEmSdFQ0FUlV9Tng/m5bkgDPBv5s1PQB4FdatiNJkiRJR8s4zkk6Bbi3quYu/rAdOGuhFZNsSrI1ydbpXT1daEWSJEmSGoxjCvCFBigueOmwqtoMbAZYeeb6Hi4vJkmSJC0OM71ckVhHYhw9SfcAJyeZK8DOBnaMYTuSJEmS1Lvei6SqKuBq4MWjplcAH+97O5IkSZI0Dk3D7ZJcAzweWJtkO/CqqroS+PfAZUn+E/A14L3NmUqSJEnHi2LRT7M9ZIddJFXV2s798x9induBpzXkJUmSJEkTMY5zkiRJkiTpmDWO2e0kSZIkNSgcbjdJ9iRJkiRJUsdgepJqCvatm2mKceLX2vM4/doDD7/SIfj285b1EmcIVnxvqjnGzPLhXAZrz2P2twe5ZXlziKHsk1ra/l+q1Xf381z2/FR7jGX3tT+f/Se0P597zmiPsfxHzSEAWHV3+z5ZeU/758DuHvbJkv3tz2Uo7z2Ak25p/xrefWYf+7U5BAdWtv/fdXpVP69NL8fJ0vZc0sPTWbK3/bms+EH7a7Pn9La/0eb0kcsJt7d9Ht29tzkFHQcGUyRJkiRJ+jGH202Ow+0kSZIkqcMiSZIkSZI6HG4nSZIkDUwRh9tN0GH3JCXZ2bn/6ST3JvnEvHVel+TWJJXk1D4SlSRJkqSjoXW43VuASxZo/6/Afwt8qzG+JEmSJB1VTUVSVX0OuH+B9q9V1R0tsSVJkiRpEjwnSZIkSRqg8pykiZno7HZJNiXZmmTr9K5dk0xFkiRJkoAJF0lVtbmqNlbVxqk1ayaZiiRJkiQBDreTJEmSBmkGh9tNSlNPUpJrgI8Cz0myPcnzRu3/Y5LtwNnAjUn+S3uqkiRJkjR+h92TVFVrO/fPf4h13gW8qyEvSZIkSZoIh9tJkiRJA1MFM85uNzETnbhBkiRJkobGIkmSJEmSOgYz3C7TsPzetprtvse053HfY1a3BwGW7OsljMZg1d8vb44xvbI9jyX7Fk8X+s6z+3kufbxvaqo9xtSe9ufTR4y+7F036QxmTT0wjH2S6WHkAbDvpPYYS3cN4/nc/9PtMaYeaI/Rl6mhHCd9pFHtIVZ8fzj/V9+/9uHXOZg+vieOFi8mOznDOeIlSZIkLSpJnp/kG0luTfL6BZYnybtGy29M8pRR+/okVye5Jcm2JL/VecwbktyZ5PrR7YWdZecm+dLoMTclWTlqf9no9xuTfDrJqQfL2yJJkiRJUu+STAF/DLwAeCLwsiRPnLfaC4DHjW6bgHeP2g8Av1NVTwCeDvzmvMe+o6o2jG5bRttbCnwIeG1V/VPgWcD+Ufv/AVxYVecCNwKvO1jugxluJ0mSJGlOFsPsdk8Dbq2q2wGSXAa8CLi5s86LgA9WVQHXJjk5yRlVdRdwF0BV3Z/kFuCseY+d758BN1bVDaPHfX+03WXMDl5dk+T7wInArQdL3J4kSZIkSeNwFvCdzu/bR22HtU6SRwM/B3y50/y60dC59yWZO/v2HKCSXJnkq0n+HUBV7Qd+HbgJ2MFsr9Z7D5a4RZIkSZKkI3Fqkq2d26Z5yxfqCps/lchB10myFvhz4Ler6r5R87uBxwIbmO1tetuofSnwS8Cvjn5elOQ5o56kX2e20DqT2eF2//PBnthhD7dLsrOq1o7uf5rZMYJfrKr/vrPOh4GNwH7gb4DXjCo4SZIkSYfgGJjd7p6q2niQ5duB9Z3fz2a2J+eQ1hkVN38OfLiqPja3QlXdPXc/yXuAT3RifaGq7hkt2wI8Bbhv9LjbRu3/L/CgSSS6WnuS3gJcskD7h4HHA08GVgH/pnE7kiRJko4tXwEel+QxSZYDFwNXzFvnCuDlo1nung78qKruShJmh8TdUlVv7z4gyRmdXy8Cvj66fyVwbpLVo8kaLmD2HKY7gScmeeRovecCtxws8aaJG6rqc0metUD7lrn7Sf6G2YpQkiRJ0nGiqg4keR2zxcsU8L6q2pbktaPllwJbgBcyO5HCbuDXRg9/BrOdMTcluX7U9vujOuPNSTYwOyzvDuA1o3g/TPJ2ZouzArZU1ScBkvyvwF8l2Q98C3jlwXIf6+x2oy6yS4Dfeojlm5id6o+lJw3kaoeSJEmSejEqarbMa7u0c7+A31zgcV/kIS6nXFULjWSbW/YhZqcBn99+KXDpgx+xsHFPAf6fgb+qqmsWWlhVm4HNACvPXN/D9aAlSZKkY1/BYpgC/Jg1tiIpyf8CPJJR95ckSZIkHQvGUiQl+TfA84DnVNXMOLYhSZIkSePQVCQluYbZWezWJtkOvKqqrmR2vN+3gC/NTkzBx6rqf2tNVpIkSTouFJQno0zMYRdJc9dIGt0//yHWGfe5TpIkSZI0Fq3XSZIkSZKkRcUeH0mSJGmAZhaeAVtHgT1JkiRJktRhkSRJkiRJHQ63kyRJkgamgPJishNjT5IkSZIkdVgkSZIkSVKHw+0kSZKkwQkzDrebmMPuSUqys3P/00nuTfKJeeu8N8kNSW5M8mdJ1j44kiRJkiQNT+twu7cAlyzQ/j9V1X9TVecC3wZe17gdSZIkSToqmoqkqvoccP8C7fcBJAmwitkJOiRJkiRp8MZ2TlKS/xt4IXAz8DsPsc4mYBPA0pPWjSsVSZIk6ZhTdjNMzNhmt6uqXwPOBG4BXkPeHVMAACAASURBVPoQ62yuqo1VtXFq9ZpxpSJJkiRJh2ysU4BX1TTwEeBfjHM7kiRJktSX3ofbjc5DemxV3Tq6/8vA3/a9HUmSJGkxK6cAn5imIinJNcDjgbVJtgOvAj4LfCDJiUCAG4Bfb01UkiRJko6Gwy6Sqmpt5/75D7HaM444I0mSJEmaoLHNbidJkiTpyFQ53G6SxjpxgyRJkiQda4bTkxSYXtk2GfzSXe3V9tLdzSEA2H9CP3GGIDPtMWpA5fjM8h4uOtDDf3aW7G9Pow811UeQHmLQz7GW6fYYM8vaY0yvat8pfXymAb28PksOtMeYXtEeo4/Xlz52a0/H/NI97TEO9HEFjR6eTx+frUv2DeeY70UPT6ePz+ipB9pjTK9sjwH9vIen9jbm0MN3jRa/4RRJkiRJkv7RjMPtJmZA/9+XJEmSpMmzSJIkSZKkDofbSZIkSQNUQzm/7jhkT5IkSZIkdTQVSUlOT3JZktuS3JxkS5Jzknw6yb1JPtFXopIkSZJ0NBzxcLskAS4HPlBVF4/aNgCnAW8BVgOv6SNJSZIk6XjjxWQnp+WcpAuB/VV16VxDVV0/dz/JsxpiS5IkSdJEtAy3exJwXcvGk2xKsjXJ1uldu1pCSZIkSVIvJjpxQ1VtrqqNVbVxak0flwyXJEmSpDYtw+22AS/uKxFJkiRJs4p4TtIEtfQkXQWsSPLquYYk5yW5oD0tSZIkSZqMIy6SqqqAi4DnjqYA3wa8AdiR5Brgo8BzkmxP8rxespUkSZKkMWsZbkdV7QBessCi81viSpIkSce7mnQCx7GJTtwgSZIkSUNjkSRJkiRJHU3D7YZm38ntnZLpaxaRRTQZyf4T2vfr0t3D2SE11R5j5XfbY+w/sT1GH5bsa4+xbGd7DIB9J7fHmFnRHiMH2mOs+EH7MT/dw3MByEx7jANr2j8Hpva075OZ5c0hetkf1dO/GPef0B6jj+fTx3fWCXe0x9h5dnsMgCX722P08V3Rx36deqA9xt51w/ken+rhO+dA41Vjenltj4bC2e0myJ4kSZIkSeqwSJIkSZKkjkU13E6SJElaNJzebmLsSZIkSZKkDoskSZIkSepoKpKSnJ7ksiS3Jbk5yZYkT0vypSTbktyY5KV9JStJkiQdL6oy6NtidsTnJCUJcDnwgaq6eNS2ATgJeHlVfTPJmcB1Sa6sqnt7yViSJEmSxqhl4oYLgf1VdelcQ1Vd312hqnYk+S7wSMAiSZIkSdLgtQy3exJw3cFWSPI0YDlw20Ms35Rka5Kt07t2NaQiSZIkSf0Y2xTgSc4A/gR4RVUteD3wqtoMbAZYedZ6JzmUJEmSRsq/jiempSdpG/DUhRYkORH4JPAfq+rahm1IkiRJ0lHVUiRdBaxI8uq5hiTnJbmA2QkdPlhVH21NUJIkSZKOpiMebldVleQi4J1JXg88ANwBXAs8EzglyStHq79y/qQOkiRJkhZWsOin2R6ypnOSqmoH8JIFFr2xJa4kSZIkTUrTxWQlSZIkabEZ2+x2kiRJko5QAQ63mxh7kiRJkiSpYzg9SQVTD7RVy1MPtKcx09ceWUTz2i/dtbj+izG1p/357D+hh0QGcozUVHuMfSe1xwB62Sc50B6jD9MrJp3Bj/XxGi/Z1/6+6SOPTLfH6EMG8v4dkl1nth8jWfCqioevj2OtFz0cJzPL2mMs2zmc7/Hp5e0xmj/nff/qEAynSJIkSZL0j7yY7OQ43E6SJEmSOiySJEmSJKnD4XaSJEnSEDncbmKaepKSnJ7ksiS3Jbk5yZYkFyS5Lsn1SbYleW1fyUqSJEnSuB1xT1KSAJcDH6iqi0dtG4CTgF+sqr1J1gJfT3JFVe3oJWNJkiRJGqOW4XYXAvur6tK5hqq6ft46K/C8J0mSJEnHkJYi6UnAdQstSLIe+CTws8DvPVQvUpJNwCaApSeta0hFkiRJWkxC1XCucXW8GUsvT1V9p6rOZbZIekWS0x5ivc1VtbGqNk6tXjOOVCRJkiTpsLQUSduApx5shVEP0jbg/IbtSJIkSdJR01IkXQWsSPLquYYk541mt1s1+n0d8AzgG21pSpIkSceZGvhtETvic5KqqpJcBLwzyeuBB4A7gL8A/s8kBQR4a1Xd1EeykiRJkjRuTReTHQ2ne8kCi97TEleSJEmSJqWpSJIkSZI0BoWz202Q1zCSJEmSpA6LJEmSJEnqGM5wu8D0yrZpMpbsa++SnNrXHAKAmeHs2WYzy9pjLNnfHqMv1cNrkwPtMYaievhXSabbY8wG6inOANRUe4zejrOh/DtspocYi+gYAVjSw2s8lO+b9PD69vF5BD19JvVwrPXyfIZyzPc0k1lv3xctORxLs7IdS7kuMkP56pQkSZKkQbBIkiRJkqSOgXTSS5IkSfpJQxlvefyxJ0mSJEmSOpqKpCSnJ7ksyW1Jbk6yJck5o2UnJrkzyR/1k6okSZIkjd8RD7dLEuBy4ANVdfGobQNwGvB3wBuBL/SRpCRJknTccXa7iWk5J+lCYH9VXTrXUFXXAyR5KrPF0qeBjU0ZSpIkSdJR1DLc7knAdfMbkywB3gb8XkNsSZIkSZqIcUzc8BvAlqr6zsOtmGRTkq1Jtk7v2jWGVCRJkiTp8LQMt9sGvHiB9l8Azk/yG8BaYHmSnVX1+vkrVtVmYDPAyrPWO+pSkiRJmuNfxxPT0pN0FbAiyavnGpKcB2yuqkdV1aOB3wU+uFCBJEmSJElDdMRFUlUVcBHw3NEU4NuANwA7espNkiRJko66luF2VNUO4CUHWf5+4P0t25AkSZKOOwVUJp3FcWscEzdIkiRJ0jHLIkmSJEmSOpqG20mSJEkaj3J2u4kZTJGUGVi2s23c5fTy9jz6iAGzz2exmHqgPUZNtcfoy5K97TFmlrXHGMoxkun2GH29vkPZJ33IgT6C9BADhjOFrEPrH6SP75yhvG+mV7THWLK/PQZADeWvmz7ee328vn2893r6HBnCa+NpPjoUDreTJEmSpI4B1POSJEmSHmQoIwGOQ/YkSZIkSRqLJM9P8o0ktyZ5/QLLk+Rdo+U3JnnKqH19kquT3JJkW5Lf6jzmDUnuTHL96PbCzrJzk3xp9JibkqwctS9PsjnJ3yX52yT/4mB525MkSZIkqXdJpoA/Bp4LbAe+kuSKqrq5s9oLgMeNbj8PvHv08wDwO1X11SQnANcl+Wznse+oqrfO295S4EPAJVV1Q5JTgLmzHf8D8N2qOifJEuARB8vdIkmSJEkaomN/lomnAbdW1e0ASS4DXgR0i6QXAR+sqgKuTXJykjOq6i7gLoCquj/JLcBZ8x473z8DbqyqG0aP+35n2b8GHj9qnwHuOVjiTcPtkpye5LIktyW5OcmWJOckme50f13Rsg1JkiRJg3Rqkq2d26Z5y88CvtP5ffuo7bDWSfJo4OeAL3eaXzcanve+JOtGbecAleTKJF9N8u9Gjz95tPyNo/aPJjntYE/siIukJAEuBz5fVY+tqicCvw+cBuypqg2j2z8/0m1IkiRJGqx7qmpj57Z53vKFusLmT0dx0HWSrAX+HPjtqrpv1Pxu4LHABmZ7m942al8K/BLwq6OfFyV5zqj9bOC/VtVTgC8BPzFUb76WnqQLgf1Vdek/Ppuq66vqmoaYkiRJkhaH7cD6zu9nAzsOdZ0ky5gtkD5cVR+bW6Gq7q6q6dGwufcwO6xvLtYXquqeqtoNbAGeAnwf2M1sBw/AR0ftD6mlSHoScN1DLFs56nK7NsmvPFSAJJvmuucO7N7VkIokSZK0uKSGfTsEXwEel+QxSZYDFwPzT8W5Anj5aJa7pwM/qqq7RqPW3gvcUlVv/4n9kpzR+fUi4Ouj+1cC5yZZPZrE4QLg5tH5Tn8JPGu03nM4+LlNY5u44VFVtSPJzwBXJbmpqm6bv9KoS24zwKoz1jsTvCRJkrRIVNWBJK9jtniZAt5XVduSvHa0/FJme3teCNzKbG/Pr40e/gzgEuCmJNeP2n6/qrYAb06ygdlheXcArxnF+2GStzNbnBWwpao+OXrsvwf+JMk7ge91trOgliJpG/DihRZU1Y7Rz9uTfJ7ZE60eVCRJkiRJWrxGRc2WeW3d03UK+M0FHvdFFj5fiaq65CDb+xCz04DPb/8W8MxDzbtluN1VwIokr55rSHJekguSrBj9fiqzVeBBu7MkSZIkddQxcFvEjrhIGlV9FwHPHU0Bvg14w2jx1iQ3AFcDfzDvglGSJEmSNFhN5ySNhtW9ZIFFT26JK0mSJEmTMq6JGyRJkiQdsUAteEqOjoKWc5IkSZIkadEZTE9SLYEDq9pirLq7PY99J7XHAJhZ1k+cIZhe1X5m3pK9w/lPyPTK9uez8nvtz6evY61VpttjrLi3PQbA3pPbYxzidRsOqo9/3C050B6jr8+RqT3tMfrIpabaY2SmhzyG83HEzLIePl/3D+MJrf12+3PZdVZPz2UYu6SXE9t7+Uzr4V/ifcSAfr5zmj9LhnJ8aNAGUyRJkiRJ6ljkM8gNmcPtJEmSJKnDIkmSJEmSOhxuJ0mSJA2Rw+0mxp4kSZIkSepoKpKSnJ7ksiS3Jbk5yZYk5yR5VJLPJLll1P7oftKVJEmSpPE64uF2SQJcDnygqi4etW0ATgPeCLypqj6bZC3Qw6StkiRJ0nHE4XYT09KTdCGwv6ounWuoquuB7wNLq+qzo7adVbW7LU1JkiRJOjpaiqQnAdct0H4OcG+SjyX5WpK3JFnwsl9JNiXZmmTr9K5dDalIkiRJUj/GMXHDUuB84HeB84CfAV650IpVtbmqNlbVxqk1a8aQiiRJkiQdnpYiaRvw1AXatwNfq6rbq+oA8BfAUxq2I0mSJB1fCqgM+7aItRRJVwErkrx6riHJecAKYF2SR46anw3c3LAdSZIkSTpqjrhIqqoCLgKeO5oCfBvwBmAHs0PtPpfkJiDAe3rIVZIkSZLG7oinAAeoqh3ASxZY9E3g3JbYkiRJ0vEsTgE+MeOYuEGSJEmSjlkWSZIkSZLU0TTcrk+ZgakH2mLsfHR7n+SSvc0hAJh6YPHM+LH83vbnMr28h0R6smxn+/PZe0ofx9riOUb2/FQ/4wGm9gxjn/SRRfXwL6jsb48BML2yPUame4hxoD1GH4ZxlM2amh5SNm3ue1x7jKX3t8cAenoT9xCjB30MtxrKew+gevjLM/tak2jP4ag5lnJdZOxJkiRJkqQOiyRJkiRJ6rBIkiRJkqQOiyRJkiRJ6rBIkiRJkqSOpjlGkpwOvBM4D9gL3AF8HPj1zmqPBy6uqr9o2ZYkSZJ0PPFispNzxEVSkgCXAx+oqotHbRuAE6pqw+j3RwC3Ap/pIVdJkiRJGruWnqQLgf1VdelcQ1VdP2+dFwOfqqrdDduRJEmSpKOm5ZykJwHXPcw6FwN/+lALk2xKsjXJ1gO7dzWkIkmSJEn96OG6xwtLcgbwZODKh1qnqjYDmwFWnbHeUZeSJEnSnMqkMzhutfQkbQOeepDlLwEur6r9DduQJEmSpKOqpUi6CliR5NVzDUnOS3LB6NeXcZChdpIkSZI0REdcJFVVARcBz01yW5JtwBuAHUkeDawHvtBDjpIkSdLxpY6B2yLWdE5SVe1gdljdQs5qiS1JkiRJk9Ay3E6SJEmSFp2xzW4nSZIkqcEiH9I2ZPYkSZIkSVLHoHqSMtP2+ANrp5tzOPmOfnbJ3nW9hBmEpXvaY0wvb4/Rlz4uObDv5MaDFVh591R7Ij2oHg756ZX9/Ktrak/7i9P6OQL97JMDq9v3yfIf9XN9jOkens/UA+0xZnr4HFhslwxZ8cP2GHtPaY9BD++bA2fsbY6xdNfK9kSAJT1cfGRmGB/Rw+lJ6Om9l/Y/1Vi2s+3xS3rIQYvfoIokSZIkSbMylCL5OORwO0mSJEnqsEiSJEmSpA6H20mSJElD5HC7iWnqSUpyepLLktyW5OYkW5Kck+TNSbYluSXJu5IsslNtJUmSJC1WR9yTNCp8Lgc+UFUXj9o2AGcAzwDOHa36ReAC4PNNmUqSJEnSUdDSk3QhsL+qLp1rqKrrgX3ASmA5sAJYBtzdkqQkSZIkHS0t5yQ9CbhufmNVfSnJ1cBdzM6q/0dVdctCAZJsAjYBLD1xEV1YSJIkSWrlOUkT0/vsdkl+FngCcDZwFvDsJM9caN2q2lxVG6tq49LVa/pORZIkSZIOW0uRtA146gLtFwHXVtXOqtoJfAp4esN2JEmSJOmoaSmSrgJWJHn1XEOS84DVwAVJliZZxuykDQsOt5MkSZL0YKnh3xazIz4nqaoqyUXAO5O8HngAuAP4t8CZwE3MjqT8dFX9ZQ+5SpIkSdLYNV1Mtqp2AC9ZYNFrWuJKkiRJ0qQ0FUmSJEmSxqQy6QyOW73PbidJkiRJxzKLJEmSJEnqGMxwu1oCB9Y0TpPRQ8lXU+0xYHHN+DHTwz5ZTPsDoFbMNMdIXwdbo+ml7S/O9Jr2/QGQH7S/ifs41qqHGDPL24Nkpp9hFn3kwu4ecukhjV4+S/rYrT19pi3b1R5o77r2J9THfl21dm97kOmV7TGAHGiPMdXDx9pMD39lzSxrj7Fs1zDyAHp57yzZ1xjkWPqb5FjKdZGxJ0mSJEmSOiySJEmSJKljMMPtJEmSJP3YYjtd4VhiT5IkSZIkdTQVSUlOT3JZktuS3JxkS5Jzkvxhkq+Pbi/tK1lJkiRJGrcjHm6XJMDlwAeq6uJR2wbgZcBTgA3ACuALST5VVff1kK8kSZJ0fHC43cS09CRdCOyvqkvnGqrqemA38IWqOlBVu4AbgOe3pSlJkiRJR0dLkfQk4LoF2m8AXpBkdZJTmS2m1i8UIMmmJFuTbJ3e1cMk/pIkSZLUqPfZ7arqM0nOA/4a+B7wJWDBy7pV1WZgM8DKs9bboShJkiRp4lp6krYBT11oQVW9qao2VNVzmb22+TcbtiNJkiQdX2p2CvAh3xazliLpKmBFklfPNSQ5L8kFSU4Z/X4ucC7wmbY0JUmSJOnoOOLhdlVVSS4C3pnk9cADwB3A64FrZie/4z7gX1XVgsPtJEmSJGloms5JqqodwEsWWPTElriSJEnScW+RD2kbsqaLyUqSJEnSYmORJEmSJEkdvU8BLkmSJKkHDrebmMEUSZmGZfelKcbM8qnmPPad3Bxilgf1T6gB9Vlmuj3Girvb3zpD2SdTD7S97wCW39P+3oPh7JM+LPtR+5OZXt5DIsDUnvbXuHp4iauPb5xF9tm6c337a9OH6iGN/d88sTnGsp4+Aw6s6SFID8daZtpjLNnfHmNmWQ8xevqLsY9caqrtgO3j80yL3yL6k0SSJEmS2g2mJ0mSJEnSjy32C7YOmT1JkiRJktRhkSRJkiRJHYdUJCU5PcllSW5LcnOSLUnOSfLpJPcm+cS89R+T5MtJvpnkI0l6Ov1YkiRJksbrYYukJAEuBz5fVY+tqicCvw+cBrwFuGSBh/0h8I6qehzwQ+BV/aUsSZIkSeNzKD1JFwL7q+rSuYaqur6qrqmqzwH3d1ceFVXPBv5s1PQB4Fd6yleSJEmSxupQiqQnAdcdRsxTgHur6sDo9+3AWYebmCRJkiRNwjimAF/oCl8LTmCYZBOwCWDZievGkIokSZJ0jHIK8Ik5lJ6kbcBTDyPmPcDJSeYKsLOBHQutWFWbq2pjVW2cWtXH5bElSZIkqc2hFElXASuSvHquIcl5SS5YaOWqKuBq4MWjplcAH29NVJIkSZKOhoctkkZFz0XAc0dTgG8D3gDsSHIN8FHgOf8/e/cfZllV3/n+/enqH9Ag4A8EFDIYlOs4jEFpOo4GER3ij5k8hhuD6AwEE0Gd4SbG3CTG6zPDE8eZTEyU0SRy24gB4wwOKoYLLYyPqEPmgrEbW7QhKm3M2DaRS0aUHwJdVd/7x9mlh6J+7FN7V9fp6vfrefZTdfbe63vW3metfc46a+11kuxO8tIm2W8Db0lyJ4N7lD64LLmXJEmSVqOCjPmymrW6J6mq9gBnz7HptHn2/yawuUO+JEmSJGlFtPoxWUmSJEk6UCzH7HaSJEmSulrlQ9rGmT1JkiRJkjRkbHqSagImD+0WY+19c/1E0+j56MNqupnt4Sd0j7Fmb/cYfZle1z3GxA/Hp6x11v1QWPtgD0Ho55z0cl57qL9rJhffZ9FsjNHXWFMbusdID+ekj/I6Vvo4njF5v9nwve4H08f1GXoqa33o4fXt45o23UOMTHWPATDxcPcYXc9JrbbriJbF2DSSJEmSJA0Zky9BDkRj9D2lJEmSJK08G0mSJEmSNMRGkiRJkjRmwsr/WGwfPyab5GVJvpbkziRvnWN7kry32X5bkuc2649L8tkkdyTZmeTXhtJcnOQ7SXY0yyuGtj07yc1Nmq8kOWjW812T5KuL5dt7kiRJkiT1LskE8MfAmcBu4ItJrqmq24d2eznwjGb5aeD9zd9J4Deq6tYkjwO2J/n0UNr3VNUfzHq+tcCfA+dW1ZeTPBHYO7T9fwfub5P3Vj1JSY5OcmWSXUluT7I1yYlJrk9yb5JrZ+1/UdMarCRPavMckiRJklaVzcCdVfXNqnoEuBJ45ax9XglcUQO3AEckOaaq7qqqWwGq6j7gDuCpizzfzwK3VdWXm3R/X1VTAEkOBd4C/Ls2GV+0kZQkwNXA56rqhKp6FvA24CjgXcC5cyT7H8A/Bf62TSYkSZIkzVJjvsCTkmwbWi6cdQRPBb499Hg3j23oLLpPkuOB5wBfGFp9UTM877Ikj2/WnQhUkhuS3Jrkt4b2fwfwh8CDtNBmuN0ZwN6qunRmRVXtGMr0i2YnqKovNdva5EGSJEnS/ueeqtq0wPa5GgOz72ZacJ+mB+jjwJur6gfN6vczaPQUP278/DKDts3PAKcyaAx9Jsl24O+Bp1fVrzcNrkW1aSSdBGxvE2xUTWvzQoC1hz9+kb0lSZIk7Ud2A8cNPT4W2NN2nyTrGDSQPlJVn5jZoaq+O/N/kg8AM7f+7AY+X1X3NNu2As9lcB/SKUm+xaD98+Qkn6uqF82X8RWd3a6qtlTVpqraNHHIISuZFUmSJEn9+iLwjCRPS7IeOAe4ZtY+1wDnNbPcPQ/4flXd1dzy80Hgjqp693CCJMcMPTwLmJmt7gbg2Uk2NpM4nA7cXlXvr6qnVNXxDHqavr5QAwna9STtBF7VYj9JkiRJfRhhmu1xVVWTSS5i0HiZAC6rqp1J3thsvxTYCrwCuJPBELnXNclfwGDug68kmbnV521VtRX4/SQnMxhu9y3gDU287yV5N4PGWQFbq+q6peS9TSPpRuDfJ7mgqj4AkORUYGNVfX4pTypJkiRp9WsaNVtnrRue66CAfz1Hur9k7vuVqKq5Jo6b2fbnDKYBn2/7txjcTrSgRYfbNRk/CzizmQJ8J3AxsCfJTcBVwEuS7E7yUoAkv5pkN4Mxhbcl+dPFnkeSJEmSxkGrH5Otqj3A2XNsOm2e/d8LvLdDviRJkqQD234+3G5/tqITN0iSJEnSuLGRJEmSJElDWg232ycKMtkxRg+/XZvp7jFWmzV7VzoH/erjNa6J7jFWk7E6H2MyNKFW2VdQmeojSA8xVpsxKa99mF630jkYYll7lF7qb096uTauonqzqAPpWMfMKnsblyRJkqRubCRJkiRJ0pDxGW4nSZIk6Uf29x+T3Z/ZkyRJkiRJQ2wkSZIkSdKQVo2kJEcnuTLJriS3J9ma5MQk1ye5N8m1s/b/SJKvJflqksuSjNOcN5IkSdL4qzFfVrFFG0lJAlwNfK6qTqiqZwFvA44C3gWcO0eyjwDPBP4xcDDw+t5yLEmSJEnLqM3EDWcAe6vq0pkVVbVj5v8kL5qdoKq2Dm3/K+DYbtmUJEmSpH2jzXC7k4DtSwneDLM7F7h+nu0XJtmWZNvUgw8s5SkkSZIkqVfLPQX4nwD/vapummtjVW0BtgAc9JTjVvnIRkmSJKmlA+C+n3HWpidpJ3DKqIGT/FvgSOAto6aVJEmSpJXSppF0I7AhyQUzK5KcmuT0+RIkeT3wUuA1VTXdPZuSJEmStG8s2kiqqgLOAs5spgDfCVwM7ElyE3AV8JIku5O8tEl2KYPZ725OsiPJv1me7EuSJEmrU2q8l9Ws1T1JVbUHOHuOTafNs/9y3+skSZIkScui1Y/JSpIkSdKBwh4fSZIkaRyt8iFt42xsGkkpSNcpHnooSOt6+rmmRw7vJ844WPtg9xiTG7vH6Ev10H86vb57YZt4KN0z0oPqIRs10T0GwJrJ7jHSQ4zpdd1jTB3cvYysfaCfMtL52gqseaR7jMlDusfo4/XtZQxFTx9cJh7qHqOP62sf9xbsPbR7kHX391Pm+7iu9WFc7tmYeLh7jMmDu8cAoIfXZm3Hz2p9XBO1+jncTpIkSZKGjE1PkiRJkqQfG5feyAORPUmSJEmSNMRGkiRJkiQNcbidJEmSNI4cbrdiWvUkJTk6yZVJdiW5PcnWJCcmuT7JvUmunbX/B5N8OcltST6W5NDlyb4kSZIk9WvRRlKSAFcDn6uqE6rqWcDbgKOAdwHnzpHs16vqp6rq2cD/BC7qMc+SJEmStGzaDLc7A9hbVZfOrKiqHTP/J3nR7ARV9YNmW4CDsbNQkiRJ0n6izXC7k4DtowZO8iHg74BnAu+bZ58Lk2xLsm3ywZ5+xVWSJEna39V+sKxiyza7XVW9DngKcAfw6nn22VJVm6pq09qNPfwUuyRJkiR11KaRtBM4ZSnBq2oK+CjwC0tJL0mSJEn7WptG0o3AhiQXzKxIcmqS0+faOQNPn/kf+Dngr/vIrCRJknQgyH6wrGaLNpKqqoCzgDObKcB3AhcDe5LcBFwFvCTJ7iQvZXDOLk/yFeArwDHA7y7XAUiSJElSn1r9mGxV7QHOnmPTafMkecGScyRJkiRJK6hVI0mSJEnSPrbKZ5AbZ8s2u50kSZIk7Y9sJEmSJEnSkLEZblcTsPfQspzrRQAAIABJREFUbn2Ke5+8t6fcdLdx1/qVzkJvJg/qHiOT3WP0ZWKqe4ypye5zumS6ez560cNXJVMb+xkPMPFQ9/NaE93zsaaHS8naHo5luqcrdNdra1/W3TcmcyH1UPf6qr9TPVxfJx7uHqN6uA6s/WH317ePugdQPRS1PvIyva57jD5em+kerotrH+weA/q5rk1u7Ja+j3O6r2Q8Lt8HpP2omEiSJEnS8rORJEmSJElDxma4nSRJkqQhDrdbMfYkSZIkSdKQVo2kJEcnuTLJriS3J9ma5MQk1ye5N8m186R7X5L7+82yJEmSJC2fRYfbJQlwNXB5VZ3TrDsZOAp4F7AReMMc6TYBR/SaW0mSJOlA4XC7FdPmnqQzgL1VdenMiqraMfN/khfNTpBkgkED6rXAWd2zKUmSJEn7RpvhdicB20eMexFwTVXdtdBOSS5Msi3JtqkHHhjxKSRJkiSpf73PbpfkKcAvAi9abN+q2gJsATjo2OPsUJQkSZK04to0knYCrxoh5nOApwN3Dm5nYmOSO6vq6UvInyRJknTgKYhdCCumzXC7G4ENSS6YWZHk1CSnz7VzVV1XVUdX1fFVdTzwoA0kSZIkSfuLRRtJVVUMJl84s5kCfCdwMbAnyU3AVcBLkuxO8tJlza0kSZIkLbNW9yRV1R7g7Dk2ndYi7aGjZkqSJEk64DncbsW0+jFZSZIkSTpQ2EiSJEmSpCG9TwEuSZIkqTtnt1s5Y9NIyhSs/0E6xVh33/rO+Vgz1TkEAFMb+okzDtbs7R5jel33GH2Z6qHUV7eiOjAu/bg9XIDX3t/HCYHq44o03T1EH/mY7H45Ij0cC8C6+/p5fbrq4zrQyznp4XT09rmlh0B9vN/08kGshxh9vVf0cY2eHpc63Mdr08P5mDy4ewygl7xksnsMaTHj8jFNkiRJksbC2PQkSZIkSRricLsVY0+SJEmSJA2xkSRJkiRJQ1o1kpIcneTKJLuS3J5ka5ITk1yf5N4k187a/8+S/E2SHc1y8vJkX5IkSVqdUuO9rGaL3pOUJMDVwOVVdU6z7mTgKOBdwEbgDXMk/c2q+liPeZUkSZKkZddm4oYzgL1VdenMiqraMfN/khctQ74kSZIkaUW0GW53ErB9CbHfmeS2JO9JMuevOCS5MMm2JNsmH3xgCU8hSZIkSf1arokbfgd4JnAq8ATgt+faqaq2VNWmqtq0duMhy5QVSZIkaT9T+8GyirVpJO0EThklaFXdVQMPAx8CNi8lc5IkSZK0r7VpJN0IbEhywcyKJKcmOX2+BEmOaf4G+Hngq10zKkmSJEn7wqITN1RVJTkLuCTJW4GHgG8Bb05yE4NhdYcm2Q38SlXdAHwkyZFAgB3AG5frACRJkqRVaZUPaRtnbWa3o6r2AGfPsem0efZ/cZdMSZIkSdJKWa6JGyRJkiRpv9SqJ0mSJEnSvhMgDrdbMfYkSZIkSdKQselJqsD0GORmaqKnQKuo5d/L67KKzgf4zc6y6eO8pocYPeQjU91j9KX6uq51lOmVzkFjldXfcTmv41TmvUY/WvXwlfi4lDOgn+u8tIgxaJZIkiRJegwb/CvG4XaSJEmSNMRGkiRJkiQNcbidJEmSNIZSjrdbKa16kpIcneTKJLuS3J5ka5ITk1yf5N4k187aP0nemeTrSe5I8qvLk31JkiRJ6teiPUlJAlwNXF5V5zTrTgaOAt4FbATeMCvZ+cBxwDOrajrJk/vMtCRJkiQtlzbD7c4A9lbVpTMrqmrHzP9JXjRHmjcBr62q6Wb/uzvmU5IkSTpwFM5ut4LaDLc7Cdg+YtwTgFcn2ZbkU0meMXrWJEmSJGnfW67Z7TYAD1XVJuADwGVz7ZTkwqYhtW3qwQeWKSuSJEmS1F6bRtJO4JQR4+4GPt78fzXw7Ll2qqotVbWpqjZNbDxkxKeQJEmSpP61aSTdCGxIcsHMiiSnJjl9gTSfBF7c/H868PWlZ1GSJEk68KTGe1nNFm0kVVUBZwFnNlOA7wQuBvYkuQm4CnhJkt1JXtok+z3gF5J8BfgPwOuXJfeSJEmS1LNWPyZbVXuAs+fYdNo8+98L/LMO+ZIkSZKkFdGqkSRJkiRpH1vlQ9rG2XLNbidJkiRJ+yUbSZIkSZI0ZGyG29UEPHJ4tz7FJ+3o3id56Hce6RwD4Nv/dEMvccbBdA+lZM1k9xh92Xvcw51jPOGm7q/vfcd3DtGLiR+mc4wn3j7VQ07g7lO6f2/Tx/FMHtLD+IYevoJKP6eVQ/+2e4zJg7uf14ee3P28ZrJ7PmpifMavrH2g+/Hs7fjeCZAertHHfab7tbWv984+ysn0uh7O63TnEKy7v4cy8rjux7L30H7qzYa/735xXLO3W/r9aVa2/Smvq409SZIkSZI0xEaSJEmSJA0Zm+F2kiRJkoY43G7F2JMkSZIkSUNsJEmSJEnSkFaNpCRHJ7kyya4ktyfZmuTEJNcnuTfJtbP2vynJjmbZk+STy5N9SZIkaRWqwex247y0keRlSb6W5M4kb51je5K8t9l+W5LnNuuPS/LZJHck2Znk14bSXJzkO0PtjVcMbXt2kpubNF9JclCSjUmuS/LXzfrfWyzfi96TlCTA1cDlVXVOs+5k4CjgXcBG4A3DaarqtKH0Hwf+YrHnkSRJkrR6JJkA/hg4E9gNfDHJNVV1+9BuLwee0Sw/Dby/+TsJ/EZV3ZrkccD2JJ8eSvueqvqDWc+3Fvhz4Nyq+nKSJwJ7gQ3AH1TVZ5OsBz6T5OVV9an58t5m4oYzgL1VdenMiqraMZSZF82XsDmgFwOva/E8kiRJklaPzcCdVfVNgCRXAq8EhhtJrwSuqKoCbklyRJJjquou4C6AqrovyR3AU2elne1ngduq6stNur9v1j8IfLZZ90iSW4FjF8p4m+F2JwHbW+w3l7OAz1TVD+bamOTCJNuSbJt64IElPoUkSZKkFfCkmc/yzXLhrO1PBb499Hh3s26kfZIcDzwH+MLQ6oua4XmXJXl8s+5EoJLckOTWJL81O8NJjgB+DvjMQge23FOAvwb40/k2VtUWYAvAhuOOc5JDSZIkacb4fzq+p6o2LbA9c6ybfVQL7pPkUODjwJuHOl7eD7yj2e8dwB8Cv8ygbfMzwKkMeo8+k2R7VX2mibUW+C/Ae2d6t+bTpidpJ3BKi/0epRkDuBm4btS0kiRJkvZ7u4Hjhh4fC+xpu0+SdQwaSB+pqk/M7FBV362qqaqaBj7AoM0xE+vzVXVPVT0IbAWeOxR7C/CNqrpksYy3aSTdCGxIcsHMiiSnJjl9kXS/CFxbVQ+1eA5JkiRJq8sXgWckeVozYcI5wDWz9rkGOK+Z5e55wPer6q5m8rgPAndU1buHEyQ5ZujhWcBXm/9vAJ7dzGa3Fjid5h6mJP8OOBx4c5uMLzrcrqoqyVnAJc20fQ8B3wLenOQm4JnAoUl2A79SVTc0Sc8BFp1eT5IkSdKjhfbTbI+rqppMchGDxssEcFlV7Uzyxmb7pQx6e14B3MlgiNzMhG8vAM4FvpJkZtK4t1XVVuD3m9m2i0G75A1NvO8leTeDxlkBW6vquiTHAv8X8NfArYP2F39UVfPeFtTqnqSq2gOcPcem0+ZYN5PmRW1iS5IkSVqdmkbN1lnrhmfNLuBfz5HuL5n7fiWq6twFnu/PGUwDPrxu93yx5tPqx2QlSZIk6UCx3LPbSZIkSVqK2s/H2+3HxqaRlClYf+9IvWCPcd9PdEsPcN8/OKhzDICJVTRdxcRKZ6BnE3d2f40fPKqHfIxLGelebfhf/7CfUtLLOenheNY+2EOQPvSUjR8e2U+criYe6uGA+vi8sLeHfPT02lQP78JrHxiP83rX87pfW8fmugismRyP8zrdQxnpo+71Un970rXe1PgcisaYw+0kSZIkacjY9CRJkiRJ+rH9fXa7/Zk9SZIkSZI0xEaSJEmSJA1xuJ0kSZI0bop+JqvRkrTqSUpydJIrk+xKcnuSrUlOTHJ9knuTXDtr/5ckuTXJjiR/meTpy5N9SZIkSerXoo2kJAGuBj5XVSdU1bOAtwFHAe8C5vrF2/cD/6KqTgb+M/D2/rIsSZIkScunzXC7M4C9VXXpzIqq2jHzf5IXzZGmgMOa/w8H9nTIoyRJkiTtM20aSScB20eM+3pga5IfAj8AnjfXTkkuBC4EWHvY40d8CkmSJGn1yvRK5+DAtVyz2/068IqqOhb4EPDuuXaqqi1VtamqNk0ccsgyZUWSJEmS2mvTSNoJnNI2YJIjgZ+qqi80qz4KPH8JeZMkSZKkfa5NI+lGYEOSC2ZWJDk1yenz7P894PAkJzaPzwTu6JZNSZIk6QBTY76sYovek1RVleQs4JIkbwUeAr4FvDnJTcAzgUOT7AZ+papuaBpUH08yzaDR9MvLdgSSJEmS1KNWPyZbVXuAs+fYdNo8+1/NYNpwSZIkSdqvtGokSZIkSdq3ssqHtI2z5ZrdTpIkSZL2SzaSJEmSJGnI+Ay3C9S6biH2buzeJzn5uKnOMQAO3j0+p7ar6fXdY6x5pHuMvvTxw2xrH+weY++Y/DTYWP1QXQ/DCmqie4w1e7vHWPvD7jH2Pq57DIDq4XI0taH7i7Pu/nSOUX18tddDOeur3uw9rIfz+v3xOK/rv989xiNHdI8B/VwHMtlDPno4r32UtekN3WOsebh7DOjnnEwe0rHe7C9dBAWU4+1Wyv5STCRJkiRpn7CRJEmSJElDVs+YMEmSJGkVcXa7lWNPkiRJkiQNadVISnJ0kiuT7Epye5KtSU5Mcn2Se5NcO2v/Fye5NclXk1yexB4rSZIkSfuFRRtJSQJcDXyuqk6oqmcBbwOOAt4FnDtr/zXA5cA5VXUS8LfAL/WdcUmSJGlVqzFfVrE2PUlnAHur6tKZFVW1o6puqqrPAPfN2v+JwMNV9fXm8aeBX+glt5IkSZK0zNo0kk4Cto8Q8x5gXZJNzeNXAcfNtWOSC5NsS7Jt6oEHRngKSZIkSVoevU/cUFUFnAO8J8lfMehpmvMn2apqS1VtqqpNE4eMyS9rSpIkSTqgtZlQYSeD3qDWqupm4DSAJD8LnDh61iRJkqQDU3AK8JXUpifpRmBDkgtmViQ5Ncnp8yVI8uTm7wbgt4FL59tXkiRJksbJoo2kZvjcWcCZzRTgO4GLgT1JbgKuAl6SZHeSlzbJfjPJHcBtwP9TVTcuT/YlSZIkqV+tfr+oqvYAZ8+x6bR59v9N4Dc75EuSJEk6cFUNFq2I3idukCRJkqT9mY0kSZIkSRrSaridJEmSpH3L2e1Wzlg1kqbXdisJB383nfOw94GeTslq6qPro4au6f7a9GVyQ/fjmXioh+MZkzLSx/V34uEeggBTB3ePkenuMaqHy8DeQ3vIR09lZM0j3WNkunuZn+7hvPbx+vZR9/q6TaCPa0kf5bUP6eGkVE/vFb3UnYkeYvRwOH0cS/Z2jzG9oXuMQaDuISYe6XhibXiohTH5mCZJkiRJ42FMvn+SJEmS9Cj2eq0Ye5IkSZIkaYiNJEmSJEka0qqRlOToJFcm2ZXk9iRbk2xOcnOSnUluS/Lqof2fluQLSb6R5KNJ1i/fIUiSJEmrT2q8l9Vs0UZSkgBXA5+rqhOq6lnA24CDgfOq6h8BLwMuSXJEk+w/Au+pqmcA3wN+ZVlyL0mSJEk9a9OTdAawt6ounVlRVTuq6vNV9Y3m8R7gbuDIplH1YuBjze6XAz/fb7YlSZIkaXm0aSSdBGxfaIckm4H1wC7gicC9VTXZbN4NPHWedBcm2ZZk29QDD7TPtSRJkiQtk85TgCc5Bvgw8EtVNd30JM0256jFqtoCbAE46KnHrfKRjZIkSVJLBUz78XiltOlJ2gmcMteGJIcB1wFvr6pbmtX3AEckmWmAHQvs6ZpRSZIkSdoX2jSSbgQ2JLlgZkWSU5OczmBChyuq6qqZbVVVwGeBVzWrfgn4i/6yLEmSJEnLZ9FGUtPoOQs4s5kCfCdwMfDCZjk/yY5mOblJ9tvAW5LcyeAepQ8uS+4lSZKk1arGfFnFWt2T1Mxed/Ycm94xz/7fBDZ3yJckSZIkrYhWPyYrSZIkSQeKzrPbSZIkSepfVvmQtnFmT5IkSZIkDRmfnqSCiYfm+oml9iYP7p6NTHWPsdqseaTb6zJuupYzgMmNPWRkXL4d6uHlnTqoewxgfM7JmOSjr+tRTfQRpHuIsflGdFzyAWRy8X32Fw89cXzeK8bmvXxcyloPL81YldXpjunH5XXRWBufRpIkSZKkHytbdCvF4XaSJEmSNMRGkiRJkiQNcbidJEmSNIbG5l7OA1CrnqQkRye5MsmuJLcn2Zpkc5Kbk+xMcluSVw/tf1GSO5NUkictX/YlSZIkqV+L9iQlCXA1cHlVndOsOxk4HDivqr6R5CnA9iQ3VNW9wP8ArgU+t2w5lyRJkqRl0Ga43RnA3qq6dGZFVe0Y3qGq9iS5GzgSuLeqvgQwaF9JkiRJGknhdOUrqM1wu5OA7QvtkGQzsB7YNcqTJ7kwybYk26YefGCUpJIkSZK0LDrPbpfkGODDwOuqaqSf96qqLVW1qao2TWw8pGtWJEmSJKmzNo2kncApc21IchhwHfD2qrqlz4xJkiRJ0kpo00i6EdiQ5IKZFUlOTXI6gwkdrqiqq5Yrg5IkSdKBJkCqxnpZzRZtJFVVAWcBZzZTgO8ELgZe2CznJ9nRLCcDJPnVJLuBY4Hbkvzpsh2BJEmSJPWo1Y/JVtUe4Ow5Nr1jnv3fC7y3Q74kSZIkaUW0aiRJkiRJ2sdGmhJNfeo8u50kSZIkrSY2kiRJkiRpyPgMtwtMr+82S8b0uu7ZOPjv0j0IsPfQXsKMhfU/6B7jkcO6x+hL9VDqJw/uPqPLuvv6KWtdTa/vHmNqQz8z3PRxTiYe6Z6PyY3dY+w9rPsYifXf6+l7rB6K2rr7u8fo4zqwpofXt3o4relpCMz6+7rHePjx3WP0MaTnoaOmOsc46O6J7hmBXsp8uh8O1UM+qofPNhv+V/cYDx/RPQbQy9fz6+/tlr6P13ZfWe0zyI0ze5IkSZIkaYiNJEmSJEkaMj7D7SRJkiQNVLNoRdiTJEmSJElDWjWSkhyd5Moku5LcnmRrks1Jbk6yM8ltSV49tP9HknwtyVeTXJakh9sOJUmSJGn5LdpIShLgauBzVXVCVT0LeBtwMHBeVf0j4GXAJUlm5j75CPBM4B83+71+OTIvSZIkrU4FNebLKtbmnqQzgL1VdenMiqraMbxDVe1JcjdwJHBvVW2d2Zbkr4Bje8qvJEmSJC2rNsPtTgK2L7RDks3AemDXrPXrgHOB65eaQUmSJEnalzpP3JDkGODDwOuqavbP0f0J8N+r6qZ50l6YZFuSbVMPPNA1K5IkSZLUWZvhdjuBV821IclhwHXA26vqllnb/i2D4XdvmC9wVW0BtgAc9NTjVvfARkmSJGkE8dPximnTk3QjsCHJBTMrkpya5HQGEzpcUVVXDSdI8nrgpcBr5uhdkiRJkqSxtWgjqaoKOAs4s5kCfCdwMfDCZjk/yY5mOblJdilwFHBzs/7fLE/2JUmSJKlfbYbbUVV7gLPn2PSOefZvFVeSJEnSPFb5NNvjrPPEDZIkSZK0mthIkiRJkqQhDouTJEmSxk1BnP5sxYxPIylQE91CHHRPOmdj6uDOIQa6Z2VsPPjU7uNh194/Riekh+G96x7o4XjG5JT0cQFed19PB9NDmKmDesjGVPcY677fvaO+rzfH6XXdYzxyWPcYa/Z2j9H1faIvfeVj76E9BOnjloUe6t5B3+3hpPR0KemlrPWRlzEZr/Pw47vHyGT3GABrHu4eY3Jj10x0z4NWP4uJJEmSJA0Zn54kSZIkST/m7HYrxp4kSZIkSRpiI0mSJEmShjjcTpIkSRpHjrZbMa16kpIcneTKJLuS3J5ka5LNSW5OsjPJbUlePbT/B5N8uVn/sSR9zN8jSZIkSctu0UZSkgBXA5+rqhOq6lnA24CDgfOq6h8BLwMuSXJEk+zXq+qnqurZwP8ELlqe7EuSJEkaV0leluRrSe5M8tY5tifJe5vttyV5brP+uCSfTXJH0ynza0NpLk7ynSQ7muUVQ9uePdSR85UkBzXrT2ke39k834IT/bcZbncGsLeqLp1ZUVU7hneoqj1J7gaOBO6tqh/MHDSDxpSdhZIkSdIBJMkE8MfAmcBu4ItJrqmq24d2eznwjGb5aeD9zd9J4Deq6tYkjwO2J/n0UNr3VNUfzHq+tcCfA+dW1ZeTPBGY+eW09wMXArcAWxl08nxqvry3GW53ErB9oR2SbAbWA7uG1n0I+DvgmcD75kl3YZJtSbZNPfBAi6xIkiRJB4ZUjfXSwmbgzqr6ZlU9AlwJvHLWPq8ErqiBW4AjkhxTVXdV1a0AVXUfcAfw1EWe72eB26rqy026v6+qqSTHAIdV1c1VVcAVwM8vFKjz7HbNk34YeF1V/ej34avqdcBTGBzQq+dKW1VbqmpTVW2aOOSQrlmRJEmStO88aabDo1kunLX9qcC3hx7v5rENnUX3SXI88BzgC0OrL2qG512W5PHNuhOBSnJDkluT/NbQc+xeJB+P0qaRtBM4Za4NSQ4DrgPe3rT8HqWqpoCPAr/Q4nkkSZIk7T/umenwaJYts7bPdd/P7C6oBfdpJoD7OPDmmVt6GAydOwE4GbgL+MNm/VrgZ4B/0fw9K8lLWubjUdo0km4ENiS5YCizpyY5ncGEDldU1VVD25Lk6TP/Az8H/HWL55EkSZI0o2q8l8XtBo4benwssKftPknWMWggfaSqPvHj01LfraqpZhTbBxgM65uJ9fmquqeqHmRw79Fzm/XHLpKPR1m0kdSM2zsLOLOZAnwncDHwwmY5f2hmiZMZtNQuT/IV4CvAMcDvLvY8kiRJklaVLwLPSPK0JOuBc4BrZu1zDXBe09HyPOD7VXVX09nyQeCOqnr3cILmdp8ZZwFfbf6/AXh2ko3NJA6nA7dX1V3AfUme18Q9D/iLhTLe6sdkq2oPcPYcm94xT5IXtIkrSZIkaXWqqskkFzFovEwAl1XVziRvbLZfyqC35xXAncCDwOua5C8AzgW+kmRmZu23VdVW4PebzpkCvgW8oYn3vSTvZtA4K2BrVV3XpH0T8GcMZt7+FAvMbActG0mSJEmS9qECphfda+w1jZqts9YN/7RQAf96jnR/ydz3ElFV5y7wfH/OYBrw2eu3MZi1u5XOs9tJkiRJ0moyNj1JmYJ19y34w7eLmlrfPR/T67rHAMgqaPnPWH9vt9cFYHpsShqsmeweo4/jyZj8xHKmuseoie4xoKd600eM7kW+l9e3t/PaQ5nv43h6qTer6NoK/bxvjcs5qR7eP/soqwBTG3oI0kOZ7+O16aX+jtF1furg7jG6npPq4Rqv1W+MPrpKkiRJAgitf7BVy8DhdpIkSZI0xEaSJEmSJA1xuJ0kSZI0jhxut2LsSZIkSZKkIa0aSUmOTnJlkl1Jbk+yNcnmJDcn2ZnktiSvniPd+5Lc33+2JUmSJGl5LDrcLkmAq4HLq+qcZt3JwOHAeVX1jSRPAbYnuaGq7m322QQcsXxZlyRJklYxh9utmDb3JJ0B7J31y7g7hneoqj1J7gaOBO5NMgG8C3gtcFaP+ZUkSZKkZdVmuN1JwPaFdkiyGVgP7GpWXQRcU1V3LZLuwiTbkmybfPCBNvmVJEmSpGXVeXa7JMcAHwZ+qaqmm6F3vwi8aLG0VbUF2AJw8DHH2Z8oSZIkacW1aSTtBF4114YkhwHXAW+vqlua1c8Bng7cObidiY1J7qyqp/eQX0mSJGn1K2B6pTNx4Goz3O5GYEOSC2ZWJDk1yekMJnS4oqqumtlWVddV1dFVdXxVHQ88aANJkiRJ0v5i0UZSVRWDyRfObKYA3wlcDLywWc5PsqNZTl7W3EqSJEnSMmt1T1JV7QHOnmPTO1qkPXTUTEmSJEkHujgF+Ipp9WOykiRJknSgsJEkSZIkSUM6TwEuSZIkaRk43G7FjFUjqdIt/cNP6j5P4vSGfgrjQXdN9BJnHFQf/Y3jVMd7yMvaB7vHmNrQPUYf1kx1j9FLGaH7NQCgeqh6aya7x1j7w+4x9vZ1R2cPr8/k+u4VZ+KhHl7gHkL0cQ1IT9e0yUO7B1r3/e4npY86vP7e7jEmN3aPAf1cB9LDtbEPfVyjp9b3kI+93WNAP2Vt6uCO9cZxVGrBYiJJkiRJQ8aqJ0mSJEkSQDncbgXZkyRJkiRJQ2wkSZIkSdKQVo2kJEcnuTLJriS3J9maZHOSm5PsTHJbklcP7f9nSf4myY5mOXn5DkGSJElaZYrBcLtxXlaxRe9JShLgauDyqjqnWXcycDhwXlV9I8lTgO1JbqiqmfltfrOqPrZcGZckSZKk5dBm4oYzgL1VdenMiqraMbxDVe1JcjdwJNDDJKCSJEmStDLaDLc7Cdi+0A5JNgPrgV1Dq9/ZDMN7T5I5fxEmyYVJtiXZNvngA60zLUmSJEnLpfPEDUmOAT4MvK6qZn7N9XeAZwKnAk8AfnuutFW1pao2VdWmtRsP6ZoVSZIkafWYHvNlFWvTSNoJnDLXhiSHAdcBb6+qW2bWV9VdNfAw8CFgcx+ZlSRJkqTl1qaRdCOwIckFMyuSnJrkdAYTOlxRVVcNJ2h6l2Ymffh54Kv9ZVmSJEmSls+iEzdUVSU5C7gkyVuBh4BvAbcALwSemOT8Zvfzm0kdPpLkSCDADuCNy5B3SZIkadXKKp9me5y1md2OqtoDnD3HpnfMs/+Lu2RKkiRJklZK54kbJEmSJGk1adWTJEmSJGkfc7jdirEnSZIkSZKGjE1PUk3A3sO6tZanDpvqnI81B092jgGQ7xzcS5xxMHlo928x1t6fHnLSj4mHu8eYXt89Rsbk9wWqh69K9vZQRgDW3de9nFQPV7V0v5T08vsR6edyxCOHd399am33GOvLMtnDAAATWklEQVR7eH2n+3jX6qG4pqcvd9c80kOMHspJH+d1soe3vb6uixMPdY/Rx3md6uG9opcy38NbcB/vnQBreri+PtL1vNo5oxbGppEkSZIkqVHAtC26leJwO0mSJEkaYiNJkiRJkoY43E6SJEkaO+XsdiuoVU9SkqOTXJlkV5Lbk2xNsjnJzUl2JrktyauH9k+Sdyb5epI7kvzq8h2CJEmSJPVn0Z6kJAGuBi6vqnOadScDhwPnVdU3kjwF2J7khqq6FzgfOA54ZlVNJ3nysh2BJEmSJPWozXC7M4C9VXXpzIqq2jG8Q1XtSXI3cCRwL/Am4LVVNd1sv7u/LEuSJEkHAIfbrZg2w+1OArYvtEOSzcB6YFez6gTg1Um2JflUkmfMk+7CZp9tUw88MEq+JUmSJGlZdJ7dLskxwIeB1830HAEbgIeqahPwAeCyudJW1Zaq2lRVmyYOOaRrViRJkiSpszaNpJ3AKXNtSHIYcB3w9qq6ZWjTbuDjzf9XA8/ukklJkiRJ2lfaNJJuBDYkuWBmRZJTk5zOoAF0RVVdNSvNJ4EXN/+fDny9j8xKkiRJB4yq8V5WsUUnbqiqSnIWcEmStwIPAd8CbgFeCDwxyfnN7uc3kzr8HvCRJL8O3A+8fhnyLkmSJEm9a/VjslW1Bzh7jk3vmGf/e4F/1iFfkiRJkrQiWjWSJEmSJO1DBUyv7iFt46zz7HaSJEmStJrYSJIkSZKkIWMz3O7h7+y+Z9dbf+NvF9jlScA9PTxVH3GM0X+MccqLMcY3L8YY37wYY3zzYozxzctqijFOeVksxj/oGH8fKfjRT5BqXxubRlJVHbnQ9iTbmh+n7aSPOMboP8Y45cUY45sXY4xvXowxvnkxxvjmZTXFGKe89HU8OrA53E6SJEmShoxNT5IkSZKkIav8B1vH2f7Uk7RljOIYo/8YfcUxRv8x+opjjP5j9BXHGP3H6CuOMfqP0VccYyxPnHGJoQNcyhaqJEmSNFYO33BUPf+Y1650NhZ0/d9esn213v/lcDtJkiRp3PhjsitqfxpuJ0mSJEnLbqwbSUnuH/r/+iT3Jrl2KTGSnJzk5iQ7k9yW5NVLiPEPkmxPsqOJ88alHEvz+LAk30nyR0uJkWSqyceOJNe0jTFHnJ9I8t+S3JHk9iTHjxIjyRlD+diR5KEkP7+EfPx+c07vSPLeJFlCjP+Y5KvNsuDr26ZsJXlaki8k+UaSjyZZv4QYFyW5M0kledIS8/GRJF9rjuuyJOuWGOeDSb7clP+PJTl0jjhHJ7kyya6mPGxNcuKo9W+eOJtHqYPzxDh9lDo43/E021rVwQXOSes6uECMkerfPHHeMEodXCAvrevgAjEWrIOjlq/MUQeXEOMxdXAJMR5TB5cQ4zH1b9QYQ7Hel0fX+VHz8mdJ/maozJy8hBhJ8s4kX2/KzK8uIcZNQ3nYk+STS4jxkiS3NjH+MsnTlxDjxU2Mrya5PMnaBc7rvNextC+vC8VoW14XitG2vC4Uo215XfS6nqHyuoR8PKasLjFOMqu8zs6nNNtY35OU5P6qOrT5/yXARuANVfXPR42RwQejqqpvJHkKsB34h1V17wgx1jM4Zw9n8AHzq8Dzq2rPKMfSPP5PwJHA/6qqi0Y5lrnijWJWnM8B76yqTzfHNF1VD44SY2jdE4A7gWNHiZHk+cC7gBc2m/4S+J2q+twIMf4Z8Gbg5cAG4PPAi6vqB4vlf76yleS/Ap+oqiuTXAp8uareP2KM5wDfAz4HbKqqR/24XcsYrwA+1Tz8z8B/H87HCHEOmzkfSd4N3F1Vvze0PcD/C1xeVZc2604GHgesnyvmPOd2vjiHA3va1MFFYtzSpg4udDxVdVObOrjIOflUmzq4SIx30LL+LXY8zeMF6+Ai5/Xf06IOLhDjlcDPME8dXEr5mqsOAueNGONRdRD4+yXk4zF1cAn5eEz9a87ZSPUtySbg14CzmmvfUs7rnwHXVtXHOrw2rwPOAM6vqukkTwb+YtTjGYr38Sb9m0bMx9eBV1bVHUn+FbAZ+N/axkiyBvhb4CVV9fUkv9s8vmyeczLvdWyE8rpQjLbldaEYbcvrQjHaltcFr+vD5bV5DUY9lj9jqKw2aUZ+j5mrvFbV3Yy5w9cfVc8/+jUrnY0FXf/t/+Q9SSutqj6T5EUd0n996P89Se5m8AFp0UbSULpHhh5uYIk9cUlOAY4CrmdwEVwRSZ4FrK2qTwNU1f2LJFnMqxh8cFy0gTRLAQcxeAMLsA747ogxngV8vqomgckkXwZeBvzXRZ98jrLVXIRfDMzcMXk5cDHwqMbJQjGa9V9q4i16AAvE2DqUr78Cjl1inOEPqwczOO/DzgD2zrzpNGl2DD33Y2LOY8E4zePF6uCiMVi8Ds4bY4Q6uFCMBZItHmMJ9a/NOVmsDs6Xl39C+zo4X4wzWbgOjlS+5qmD7wV+OEoZnaMOjlzO56iDL1hCjNn17ydHjZFkgsEXSq9l8KFzScczh6XEeBPw2qqabh6ftNR8JHkcg9f6Q0uIUcBhzf+HMyi7o8R4IvDw0GeETwO/A/zNQnGaxz+6jiX5PiOW19kxgHtHLa/zxBipvM4TY6TyOleMOcprH+8NSzonzCqv+0MD6UfGuDNjtRvr4XbLJclmBh8Gdi0h7XFJbgO+DfzHatGLNCv9GuAPgd8c9blnOSjJtiS3pOXwtjmcyOBC9okkX0ryruaitlTnAP9l1ERVdTPwWeCuZrmhqu4YMcyXgZcn2ZjBEIUzgONGzcuQJzJ4o5hsHu8GntohXmcZDLM7l8EH+6XG+BDwd8AzgffN2nwSg2/eulo0Tos6OG+MEergnDFGrIMLHUvbOjhfjFHrX5vXZ7E6OGeMEevgfPlYrA6OWr7mq4Ndy+iSy/lQHbxnKTFm1b+vLSHGRcA1VXXX0LqlHs87MxiS9B7gp5YQ4wTg1U0d+BSD13upr81ZwGcYfBAfNcbrga1JdjN4bXaMGOMeYF3T4wGDLxqOY/Tr2JLKa5dr4WIxRimvc8UYtbzOEWN2eV3qsfyorCbZsMQ4jyqvSZ6xUHoJDsBGUpJjgA8Drxv6Bqy1qvp2VT0beDrwS0mOGjHEvwK2VtW3R33uWX6i6d58LXBJkhOWEGMtcBrwfwKnMniDOn8pmWnO6z8GblhC2qcD/5BBD8lTgRcneeHCqR6tqv4bsJVBF/x/AW4GJhdMtEi25nqaDvH68CcMhtrdtNQAVfU64CnAHUDr+/L6ZB38kd7qHxwwdXCl/QmDoUvfXEriWfXvuaOkbYYP/SKP/XJjKX6HwQffU4EnAP90CTE2AA81deAD/LgHZSlewxK+YGv8OvCKqjqWQU/UWYvs/yhVVQy+XHhP0+tyHy3K7RzXsZHLa9drYYsYrcrrfDFGKa+zYyylvM6Tj9ll9beXGGd2eb2sbb504DqgGklJDgOuA95eVbd0idV8e72TwYecUfwT4KIk3wL+ADgvye8tnGTe56eqvslg7PJzRo3B4JuuL1XVN5tvvz7JiG/cQ84Grq6qvUtIexaD+0zub4YcfQp43qhBquqdVXVyVZ3J4A3rG0vIy4x7gCPS3MDL4MPjSL2GfUrybxkMG3hL11hVNQV8FPiFWZt2Aqd0jb9QnBHq4KJ5aVEH54sxSh2cNx8j1MH5Yoxa/xY7J23q4HwxRqmDC52ThergqOVrrjr4nRFjzGVJ5XxWHVxyXRmqfz85YoznMPhi4M6m7G5McudS8lJVd9XAwwwaFk8YNQaD8vvx5v+rGQxfXcp5fSKD+4iuY8RjSXIk8FNV9YVm1UcZfKgf9XzcXFWnVdVmBo2KbyyUl3muYyOV1z6uhQvFaFteF8tHm/I6T4zHlFcGDZyR8jFHWd28xOOZXV6fPVf6sVQ13ssqdsA0kjKYdOFq4IqqumqJMY5NcnDz/+MZjPP92igxqupfVNVPVNXxDL5BvqKq3jpiPh7fdDnTDGt5AXD7KDEaXwQe37zRwGA89VLiQLdvAv8ncHqStc3wgNMZfHPVWpKJ5s2WJM9mcAH8b0vMz8y3i59lMPQC4JcY3FS8zyV5PfBS4DUdvm1M01swM8b854C/nrXbjcCGJBcMpTs1yekjPt1CcdrWwXljjFAH54wBbBmhDi6Uj7Z1cL58bGC0+rfY69OmDs6Xl420r4MLnZOF6uBI5WueOvihUWKMmv/5EsxRB0eKMU/9+x+jxKiq66rq6Ko6vim7D1bV05d4PMcM5eXnGUyyMep5/SSDMgs/Li9LeW1+kcGN+Q8t4Vi+BxyeZsZK4EwGw7BGPR9Pbv5uYPBB/tJF8vKY69gSymvna+F8MUYsr4+JsYTyOtf5eEx5ZdBwHPVYZpfVry7lnPDY8vp1pEXsT7Pb3cSgy/VQBrO9/EpVLTqsJD+e/exfMrhg7RzafH499sbnhWKcyeBehmLwLekfVdWWUY9laN35DGY8G2l2uwxmg/u/gWkGDd1LquqDbWLMzsvQMYXBm8uF9egJKtrEOJ7BBfS4UT7EDx3PBINhAS9kcG6vr6pWPSZDMQ4Cbm1W/wB440KvbZuyleQngSsZfMv6JeBfNt9mjRLjV4HfAo5mMDvQ1qp6/YgxJhnMtnRfk+wTVfW7oxwPg5uRb2Jwg3MY3D/yppo1+18GQyQuYfAt3UPAtxjMGnjZXHlb4PzOFecW4N/Qsg7OE+OTwP9Byzo43/FU1Tea7eezSB2cJ8bvA39Eyzq4wHk9nhHq3wJx9tKyDs4T4y3N0qoOzhPjrQy+cYZ56uCo5WuuOsjg3o9RYjymDjIoh6PEeEwdBP60bQzmqX/NPkuqb7Pq/Kjn9UYGvQxhcA/PG5u8jRLjCOAjwE8A9zcx/r9RjyeDGVZ/r6quX+KxnAX8LoO6+D3gl5t0o8R4F/DPGdTl91fVJQvkZd7r2AjldaEYbcvrQjHaltc5YwC30b68trquD71Xj3pOH1NWq+r+JcR5THmtqi8z5g5ff1Q9/8krMjK+teu/875VO7vdWDeSJEmSpAPR4eufXM8/cswbSXv+aNU2kg6Y4XaSJEmS1IaNJEmSJEkast/8mKwkSZJ0wChgeknzNakH9iRJkiRJ0hAbSZIkSZI0xOF2kiRJ0jhyFuoVY0+SJEmSJA2xkSRJkiRJQ2wkSZIkSdIQ70mSJEmSxpH3JK0Ye5IkSZIkaYiNJEmSJEka4nA7SZIkaewUTDvcbqXYkyRJkiRJQ2wkSZIkSdIQh9tJkiRJ46aganqlc3HAsidJkiRJkobYSJIkSZKkIQ63kyRJksaRs9utGHuSJEmSJGmIjSRJkiRJGuJwO0mSJGkclcPtVoo9SZIkSZI0xEaSJEmSJA1xuJ0kSZI0bqpg2h+TXSn2JEmSJEnSEBtJkiRJkpZFkpcl+VqSO5O8dY7tSfLeZvttSZ7brD8uyWeT3JFkZ5JfG0pzcZLvJNnRLK9o1h+f5IdD6y8dSvOaJF9pnuP6JE9aKN8Ot5MkSZLUuyQTwB8DZwK7gS8muaaqbh/a7eXAM5rlp4H3N38ngd+oqluTPA7YnuTTQ2nfU1V/MMfT7qqqk2flYy3wn4BnVdU9SX4fuAi4eL6825MkSZIkjaOq8V4Wtxm4s6q+WVWPAFcCr5y1zyuBK2rgFuCIJMdU1V1VdevgNNR9wB3AU5d4JtMshyQJcBiwZ6EENpIkSZIkLcWTkmwbWi6ctf2pwLeHHu/msQ2dRfdJcjzwHOALQ6svaobOXZbk8UPrn5bkS0k+n+Q0gKraC7wJ+AqDxtGzgA8udGA2kiRJkiQtxT1VtWlo2TJre+ZIM7sLasF9khwKfBx4c1X9oFn9fuAE4GTgLuAPm/V3AT9RVc8B3gL85ySHJVnHoJH0HOApwG3A7yx0YN6TJEmSJI2h2v+nAN8NHDf0+FgeO8xt3n2axs3HgY9U1Sdmdqiq7878n+QDwLXN+oeBh5v/tyfZBZxI0xCrql1Nmv8KPGYSiWH2JEmSJElaDl8EnpHkaUnWA+cA18za5xrgvGaWu+cB36+qu5p7hz4I3FFV7x5OkOSYoYdnAV9t1h/ZTBZBkp9kMBnEN4HvAM9KcmST5kwG9zjNy54kSZIkSb2rqskkFwE3wP/f3h3iZhUFYQD9JjW4ClgIAkHSBE1gDSStomvAUckKUBj20BAknoSE1CGKJ8iaFgbRX4wqmHJfk3Pku7nJ2C8zd172krzr7rOqOt6dv01ymuRZkm9JLpIc7a4fJHmR5GtVfdl9e9Xdp0neVNXDXI/lnSd5uTt/kuSkqq6S/Epy3N0/k6SqXif5VFWXSb4nObyp9up/20wBAAD8J/t79/vxveery7jRx4v3n7v70eo6boNxOwAAgEFIAgAAGLxJAgCArekkvz2LWUUnCQAAYBCSAAAABuN2AACwRX3nfyZ7Z+kkAQAADEISAADAICQBAAAM3iQBAMDGdJK2AnwZnSQAAIBBSAIAABiM2wEAwNZ0WwG+kE4SAADAICQBAAAMxu0AAGCDbLdbRycJAABgEJIAAAAG43YAALBFttsto5MEAAAwCEkAAABDdduaAQAAW1JVH5I8WF3HX/zo7qeri7gNQhIAAMBg3A4AAGAQkgAAAAYhCQAAYBCSAAAABiEJAABg+AMr5yFoqn38OQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x1080 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_attention(attention)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
